@naturalcycles/nodejs-lib 12.56.0 → 12.59.0

package/dist/stream/readable/readableMap.js

@@ -1,24 +1,29 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.readableMap = void 0;
-const index_1 = require("../../index");
+const stream_1 = require("stream");
+const js_lib_1 = require("@naturalcycles/js-lib");
 function readableMap(readable, mapper) {
-    const out = (0, index_1.readableCreate)();
-    void (async () => {
-        try {
-            let index = 0;
-            for await (const item of readable) {
-                const v = await mapper(item, index++);
-                out.push(v);
+    let i = -1;
+    // todo: check if we need to handle errors somehow specifically
+    return readable.pipe(new stream_1.Transform({
+        objectMode: true,
+        async transform(chunk, _enc, cb) {
+            try {
+                const r = await mapper(chunk, ++i);
+                if (r === js_lib_1.SKIP) {
+                    cb();
+                }
+                else {
+                    // _assert(r !== END, `readableMap END not supported`)
+                    cb(null, r);
+                }
             }
-            // We're done
-            out.push(null);
-        }
-        catch (err) {
-            console.error(err);
-            out.emit('error', err);
-        }
-    })();
-    return out;
+            catch (err) {
+                console.error(err);
+                cb(err);
+            }
+        },
+    }));
 }
 exports.readableMap = readableMap;

package/dist/stream/stream.util.d.ts

@@ -0,0 +1,4 @@
+/// <reference types="node" />
+import { Readable } from 'stream';
+import { CommonLogger } from '@naturalcycles/js-lib';
+export declare function pipelineClose(name: string, readableDownstream: Readable, sourceReadable: Readable | undefined, streamDone: Promise<void> | undefined, logger: CommonLogger): void;

package/dist/stream/stream.util.js

@@ -0,0 +1,24 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.pipelineClose = void 0;
+function pipelineClose(name, readableDownstream, sourceReadable, streamDone, logger) {
+    readableDownstream.push(null); // this closes the stream, so downstream Readable will receive `end` and won't write anything
+    if (!sourceReadable) {
+        logger.warn(`${name} sourceReadable is not provided, readable stream will not be stopped`);
+    }
+    else {
+        logger.log(`${name} is calling readable.unpipe() to pause the stream`);
+        sourceReadable.unpipe(); // it is expected to pause the stream
+        if (!streamDone) {
+            logger.log(`${name} streamDone is not provided, will do readable.destroy right away`);
+            sourceReadable.destroy();
+        }
+        else {
+            void streamDone.then(() => {
+                logger.log(`${name} streamDone, calling readable.destroy()`);
+                sourceReadable.destroy(); // this throws ERR_STREAM_PREMATURE_CLOSE
+            });
+        }
+    }
+}
+exports.pipelineClose = pipelineClose;

package/dist/stream/transform/transformLimit.d.ts

@@ -1,5 +1,40 @@
+/// <reference types="node" />
+import { Readable } from 'stream';
+import { CommonLogger } from '@naturalcycles/js-lib';
+import { AbortableTransform } from '../../index';
 import { TransformOptions, TransformTyped } from '../stream.model';
+export interface TransformLimitOptions extends TransformOptions {
+    /**
+     * Nullish value (e.g 0 or undefined) would mean "no limit"
+     */
+    limit?: number;
+    /**
+     * If provided (recommended!) - it will call readable.destroy() on limit.
+     * Without it - it will only stop the downstream consumers, but won't stop
+     * the Readable ("source" of the stream).
+     * It is almost always crucial to stop the Source too, so, please provide the Readable here!
+     */
+    sourceReadable?: Readable;
+    /**
+     * Please provide it (a Promise that resolves when the Stream is done, e.g finished consuming things)
+     * to be able to wait for Consumers before calling `readable.destroy`.
+     * Has no effect if `readable` is not provided.
+     */
+    streamDone?: Promise<void>;
+    logger?: CommonLogger;
+    /**
+     * Set to true to enable additional debug messages, e.g it'll log
+     * when readable still emits values after the limit is reached.
+     */
+    debug?: boolean;
+}
+/**
+ * Class only exists to be able to do `instanceof TransformLimit`
+ * and to set sourceReadable+streamDone to it in `_pipeline`.
+ */
+export declare class TransformLimit extends AbortableTransform {
+}
 /**
  * 0 or falsy value means "no limit"
  */
-export declare function transformLimit<IN>(limit?: number, opt?: TransformOptions): TransformTyped<IN, IN>;
+export declare function transformLimit<IN>(opt?: TransformLimitOptions): TransformTyped<IN, IN>;

package/dist/stream/transform/transformLimit.js

@@ -1,28 +1,46 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.transformLimit = void 0;
-const stream_1 = require("stream");
+exports.transformLimit = exports.TransformLimit = void 0;
+const index_1 = require("../../index");
+const stream_util_1 = require("../stream.util");
+/**
+ * Class only exists to be able to do `instanceof TransformLimit`
+ * and to set sourceReadable+streamDone to it in `_pipeline`.
+ */
+class TransformLimit extends index_1.AbortableTransform {
+}
+exports.TransformLimit = TransformLimit;
 /**
  * 0 or falsy value means "no limit"
  */
-function transformLimit(limit, opt = {}) {
-    let index = 0;
+function transformLimit(opt = {}) {
+    const { logger = console, limit, debug } = opt;
+    if (!limit) {
+        // No limit - returning pass-through transform
+        return (0, index_1.transformNoOp)();
+    }
+    let i = 0; // so we start first chunk with 1
     let ended = false;
-    return new stream_1.Transform({
+    return new TransformLimit({
         objectMode: true,
         ...opt,
-        transform(chunk, _encoding, cb) {
-            index++;
-            if (!ended) {
-                cb(null, chunk); // pass through the item
+        transform(chunk, _, cb) {
+            i++;
+            if (i === limit) {
+                ended = true;
+                logger.log(`transformLimit of ${limit} reached`);
+                this.push(chunk);
+                (0, stream_util_1.pipelineClose)('transformLimit', this, opt.sourceReadable || this.sourceReadable, opt.streamDone || this.streamDone, logger);
+                cb(); // after pause
             }
-            else {
-                cb(null); // pass-through empty
+            else if (!ended) {
+                cb(null, chunk);
             }
-            if (limit && index === limit) {
-                ended = true;
-                console.log(`transformLimit: limit of ${limit} reached`);
-                // this.emit('end') // this makes it "halt" on Node 14 lts
+            else {
+                if (debug)
+                    logger.log(`transformLimit.transform after limit`, i);
+                // If we ever HANG (don't call cb) - Node will do process.exit(0) to us
+                cb(); // ended, don't emit anything
             }
         },
     });

package/dist/stream/transform/transformLogProgress.d.ts

@@ -1,4 +1,4 @@
-import { AnyObject } from '@naturalcycles/js-lib';
+import { AnyObject, CommonLogger } from '@naturalcycles/js-lib';
 import { TransformOptions, TransformTyped } from '../stream.model';
 export interface TransformLogProgressOptions<IN = any> extends TransformOptions {
     /**
@@ -69,6 +69,7 @@ export interface TransformLogProgressOptions<IN = any> extends TransformOptions
      * @default 1000
      */
     logEvery?: number;
+    logger?: CommonLogger;
     /**
      * Function to return extra properties to the "progress object".
      *

package/dist/stream/transform/transformLogProgress.js

@@ -15,7 +15,7 @@ const inspectOpt = {
  * Pass-through transform that optionally logs progress.
  */
 function transformLogProgress(opt = {}) {
-    const { metric = 'progress', heapTotal: logHeapTotal = false, heapUsed: logHeapUsed = false, rss: logRss = true, peakRSS: logPeakRSS = true, logRPS = true, logEvery = 1000, batchSize = 1, extra, } = opt;
+    const { metric = 'progress', heapTotal: logHeapTotal = false, heapUsed: logHeapUsed = false, rss: logRss = true, peakRSS: logPeakRSS = true, logRPS = true, logEvery = 1000, batchSize = 1, extra, logger = console, } = opt;
     const logProgress = opt.logProgress !== false && logEvery !== 0; // true by default
     const logEvery10 = logEvery * 10;
     const started = Date.now();
@@ -54,7 +54,7 @@ function transformLogProgress(opt = {}) {
         const rps10 = Math.round(sma.push(lastRPS));
         if (mem.rss > peakRSS)
             peakRSS = mem.rss;
-        console.log((0, util_1.inspect)({
+        logger.log((0, util_1.inspect)({
             [final ? `${metric}_final` : metric]: batchedProgress,
             ...(extra ? extra(chunk, progress) : {}),
             ...(logHeapUsed ? { heapUsed: (0, js_lib_1._mb)(mem.heapUsed) } : {}),
@@ -76,10 +76,10 @@ function transformLogProgress(opt = {}) {
             if (perHour > 900) {
                 perHour = Math.round(perHour / 1000) + 'K';
             }
-            console.log(`${(0, colors_1.dimGrey)((0, time_lib_1.dayjs)().toPretty())} ${(0, colors_1.white)(metric)} took ${(0, colors_1.yellow)((0, js_lib_1._since)(started))} so far to process ${(0, colors_1.yellow)(batchedProgress)} rows, ~${(0, colors_1.yellow)(perHour)}/hour`);
+            logger.log(`${(0, colors_1.dimGrey)((0, time_lib_1.dayjs)().toPretty())} ${(0, colors_1.white)(metric)} took ${(0, colors_1.yellow)((0, js_lib_1._since)(started))} so far to process ${(0, colors_1.yellow)(batchedProgress)} rows, ~${(0, colors_1.yellow)(perHour)}/hour`);
         }
         else if (final) {
-            console.log(`${(0, colors_1.boldWhite)(metric)} took ${(0, colors_1.yellow)((0, js_lib_1._since)(started))} to process ${(0, colors_1.yellow)(batchedProgress)} rows with total RPS of ${(0, colors_1.yellow)(rpsTotal)}`);
+            logger.log(`${(0, colors_1.boldWhite)(metric)} took ${(0, colors_1.yellow)((0, js_lib_1._since)(started))} to process ${(0, colors_1.yellow)(batchedProgress)} rows with total RPS of ${(0, colors_1.yellow)(rpsTotal)}`);
         }
     }
 }

package/dist/stream/transform/transformMap.d.ts

@@ -1,4 +1,4 @@
-import { AsyncMapper, AsyncPredicate, CommonLogger, ErrorMode } from '@naturalcycles/js-lib';
+import { AbortableAsyncMapper, AsyncPredicate, CommonLogger, ErrorMode } from '@naturalcycles/js-lib';
 import { TransformTyped } from '../stream.model';
 export interface TransformMapOptions<IN = any, OUT = IN> {
     /**
@@ -37,10 +37,6 @@ export interface TransformMapOptions<IN = any, OUT = IN> {
      * @default `stream`
      */
     metric?: string;
-    /**
-     * If defined - called BEFORE `final()` callback is called.
-     */
-    beforeFinal?: () => any;
     logger?: CommonLogger;
 }
 export declare function notNullishPredicate(item: any): boolean;
@@ -56,4 +52,4 @@ export declare function notNullishPredicate(item: any): boolean;
  *
  * If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
  */
-export declare function transformMap<IN = any, OUT = IN>(mapper: AsyncMapper<IN, OUT>, opt?: TransformMapOptions<IN, OUT>): TransformTyped<IN, OUT>;
+export declare function transformMap<IN = any, OUT = IN>(mapper: AbortableAsyncMapper<IN, OUT>, opt?: TransformMapOptions<IN, OUT>): TransformTyped<IN, OUT>;

package/dist/stream/transform/transformMap.js

@@ -1,13 +1,16 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.transformMap = exports.notNullishPredicate = void 0;
-const stream_1 = require("stream");
 const js_lib_1 = require("@naturalcycles/js-lib");
+const through2Concurrent = require("through2-concurrent");
 const colors_1 = require("../../colors");
+const stream_util_1 = require("../stream.util");
 function notNullishPredicate(item) {
     return item !== undefined && item !== null;
 }
 exports.notNullishPredicate = notNullishPredicate;
+// doesn't work, cause here we don't construct our Transform instance ourselves
+// export class TransformMap extends AbortableTransform {}
 /**
  * Like pMap, but for streams.
  * Inspired by `through2`.
@@ -21,24 +24,16 @@ exports.notNullishPredicate = notNullishPredicate;
  * If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
  */
 function transformMap(mapper, opt = {}) {
-    const { concurrency = 16, predicate = notNullishPredicate, errorMode = js_lib_1.ErrorMode.THROW_IMMEDIATELY, flattenArrayOutput, onError, beforeFinal, metric = 'stream', logger = console, } = opt;
+    const { concurrency = 16, predicate = notNullishPredicate, errorMode = js_lib_1.ErrorMode.THROW_IMMEDIATELY, flattenArrayOutput, onError, metric = 'stream', logger = console, } = opt;
     let index = -1;
-    let isRejected = false;
+    let isSettled = false;
     let errors = 0;
     const collectedErrors = []; // only used if errorMode == THROW_AGGREGATED
-    const q = new js_lib_1.PQueue({
-        concurrency,
-        resolveOn: 'start',
-        // debug: true,
-    });
-    return new stream_1.Transform({
-        objectMode: true,
+    return through2Concurrent.obj({
+        maxConcurrency: concurrency,
         async final(cb) {
-            // console.log('transformMap final', {index}, q.inFlight, q.queueSize)
-            // wait for the current inFlight jobs to complete and push their results
-            await q.onIdle();
-            logErrorStats(logger, true);
-            await beforeFinal?.(); // call beforeFinal if defined
+            // console.log('transformMap final')
+            logErrorStats(true);
             if (collectedErrors.length) {
                 // emit Aggregated error
                 cb(new js_lib_1.AggregatedError(collectedErrors));
@@ -48,48 +43,51 @@ function transformMap(mapper, opt = {}) {
                 cb();
             }
         },
-        async transform(chunk, _encoding, cb) {
-            index++;
-            // console.log('transform', {index})
-            // Stop processing if THROW_IMMEDIATELY mode is used
-            if (isRejected && errorMode === js_lib_1.ErrorMode.THROW_IMMEDIATELY)
-                return cb();
-            // It resolves when it is successfully STARTED execution.
-            // If it's queued instead - it'll wait and resolve only upon START.
-            await q.push(async () => {
-                try {
-                    const currentIndex = index; // because we need to pass it to 2 functions - mapper and predicate. Refers to INPUT index (since it may return multiple outputs)
-                    const res = await mapper(chunk, currentIndex);
-                    const passedResults = await (0, js_lib_1.pFilter)(flattenArrayOutput && Array.isArray(res) ? res : [res], async (r) => await predicate(r, currentIndex));
-                    passedResults.forEach(r => this.push(r));
-                }
-                catch (err) {
-                    logger.error(err);
-                    errors++;
-                    logErrorStats(logger);
-                    if (onError) {
-                        try {
-                            onError(err, chunk);
-                        }
-                        catch { }
-                    }
-                    if (errorMode === js_lib_1.ErrorMode.THROW_IMMEDIATELY) {
-                        isRejected = true;
-                        // Emit error immediately
-                        // return cb(err as Error)
-                        return this.emit('error', err);
-                    }
-                    if (errorMode === js_lib_1.ErrorMode.THROW_AGGREGATED) {
-                        collectedErrors.push(err);
-                    }
+    }, async function transformMapFn(chunk, _, cb) {
+        index++;
+        // console.log({chunk, _encoding})
+        // Stop processing if isSettled (either THROW_IMMEDIATELY was fired or END received)
+        if (isSettled)
+            return cb();
+        try {
+            const currentIndex = index; // because we need to pass it to 2 functions - mapper and predicate. Refers to INPUT index (since it may return multiple outputs)
+            const res = await mapper(chunk, currentIndex);
+            const passedResults = await (0, js_lib_1.pFilter)(flattenArrayOutput && Array.isArray(res) ? res : [res], async (r) => {
+                if (r === js_lib_1.END) {
+                    isSettled = true; // will be checked later
+                    return false;
                 }
+                return r !== js_lib_1.SKIP && (await predicate(r, currentIndex));
             });
-            // Resolved, which means it STARTED processing
-            // This means we can take more load
+            passedResults.forEach(r => this.push(r));
+            if (isSettled) {
+                logger.log(`transformMap END received at index ${index}`);
+                (0, stream_util_1.pipelineClose)('transformMap', this, this.sourceReadable, this.streamDone, logger);
+            }
+            cb(); // done processing
+        }
+        catch (err) {
+            logger.error(err);
+            errors++;
+            logErrorStats();
+            if (onError) {
+                try {
+                    onError(err, chunk);
+                }
+                catch { }
+            }
+            if (errorMode === js_lib_1.ErrorMode.THROW_IMMEDIATELY) {
+                isSettled = true;
+                return cb(err); // Emit error immediately
+            }
+            if (errorMode === js_lib_1.ErrorMode.THROW_AGGREGATED) {
+                collectedErrors.push(err);
+            }
+            // Tell input stream that we're done processing, but emit nothing to output - not error nor result
             cb();
-        },
+        }
     });
-    function logErrorStats(logger, final = false) {
+    function logErrorStats(final = false) {
         if (!errors)
             return;
         logger.log(`${metric} ${final ? 'final ' : ''}errors: ${(0, colors_1.yellow)(errors)}`);

package/dist/stream/transform/transformMapSimple.d.ts

@@ -1,4 +1,4 @@
-import { ErrorMode, Mapper } from '@naturalcycles/js-lib';
+import { CommonLogger, ErrorMode, Mapper } from '@naturalcycles/js-lib';
 import { TransformTyped } from '../stream.model';
 export interface TransformMapSimpleOptions {
     /**
@@ -7,6 +7,7 @@ export interface TransformMapSimpleOptions {
      * @default ErrorMode.THROW_IMMEDIATELY
      */
     errorMode?: ErrorMode.THROW_IMMEDIATELY | ErrorMode.SUPPRESS;
+    logger?: CommonLogger;
 }
 /**
  * Simplest version of `transformMap`.

package/dist/stream/transform/transformMapSimple.js

@@ -14,7 +14,7 @@ const js_lib_1 = require("@naturalcycles/js-lib");
  */
 function transformMapSimple(mapper, opt = {}) {
     let index = -1;
-    const { errorMode = js_lib_1.ErrorMode.THROW_IMMEDIATELY } = opt;
+    const { errorMode = js_lib_1.ErrorMode.THROW_IMMEDIATELY, logger = console } = opt;
     return new stream_1.Transform({
         objectMode: true,
         transform(chunk, _encoding, cb) {
@@ -22,7 +22,7 @@ function transformMapSimple(mapper, opt = {}) {
                 cb(null, mapper(chunk, ++index));
             }
             catch (err) {
-                console.error(err);
+                logger.error(err);
                 if (errorMode === js_lib_1.ErrorMode.SUPPRESS) {
                     cb(); // suppress the error
                 }

package/dist/stream/transform/transformMapSync.d.ts

@@ -1,4 +1,4 @@
-import { ErrorMode, Mapper, Predicate } from '@naturalcycles/js-lib';
+import { CommonLogger, ErrorMode, Mapper, Predicate } from '@naturalcycles/js-lib';
 import { TransformTyped } from '../stream.model';
 export interface TransformMapSyncOptions<IN = any, OUT = IN> {
     /**
@@ -34,6 +34,7 @@ export interface TransformMapSyncOptions<IN = any, OUT = IN> {
      * @default `stream`
      */
     metric?: string;
+    logger?: CommonLogger;
 }
 /**
  * Sync (not async) version of transformMap.

package/dist/stream/transform/transformMapSync.js

@@ -11,7 +11,7 @@ const transformMap_1 = require("./transformMap");
  */
 function transformMapSync(mapper, opt = {}) {
     let index = -1;
-    const { predicate = transformMap_1.notNullishPredicate, errorMode = js_lib_1.ErrorMode.THROW_IMMEDIATELY, flattenArrayOutput = false, onError, metric = 'stream', objectMode = true, } = opt;
+    const { predicate = transformMap_1.notNullishPredicate, errorMode = js_lib_1.ErrorMode.THROW_IMMEDIATELY, flattenArrayOutput = false, onError, metric = 'stream', objectMode = true, logger = console, } = opt;
     let isRejected = false;
     let errors = 0;
     const collectedErrors = []; // only used if errorMode == THROW_AGGREGATED
@@ -39,7 +39,7 @@ function transformMapSync(mapper, opt = {}) {
                 }
             }
             catch (err) {
-                console.error(err);
+                logger.error(err);
                 errors++;
                 logErrorStats();
                 if (onError) {
@@ -75,7 +75,7 @@ function transformMapSync(mapper, opt = {}) {
     function logErrorStats(final = false) {
         if (!errors)
             return;
-        console.log(`${metric} ${final ? 'final ' : ''}errors: ${(0, colors_1.yellow)(errors)}`);
+        logger.log(`${metric} ${final ? 'final ' : ''}errors: ${(0, colors_1.yellow)(errors)}`);
     }
 }
 exports.transformMapSync = transformMapSync;

package/dist/stream/transform/transformTap.d.ts

@@ -1,9 +1,12 @@
-import { AsyncMapper } from '@naturalcycles/js-lib';
+import { AsyncMapper, CommonLogger } from '@naturalcycles/js-lib';
 import { TransformOptions, TransformTyped } from '../stream.model';
+export interface TransformTapOptions extends TransformOptions {
+    logger?: CommonLogger;
+}
 /**
  * Similar to RxJS `tap` - allows to run a function for each stream item, without affecting the result.
  * Item is passed through to the output.
  *
  * Can also act as a counter, since `index` is passed to `fn`
  */
-export declare function transformTap<IN>(fn: AsyncMapper<IN, any>, opt?: TransformOptions): TransformTyped<IN, IN>;
+export declare function transformTap<IN>(fn: AsyncMapper<IN, any>, opt?: TransformTapOptions): TransformTyped<IN, IN>;

package/dist/stream/transform/transformTap.js

@@ -9,6 +9,7 @@ const stream_1 = require("stream");
  * Can also act as a counter, since `index` is passed to `fn`
  */
 function transformTap(fn, opt = {}) {
+    const { logger = console } = opt;
     let index = 0;
     return new stream_1.Transform({
         objectMode: true,
@@ -19,7 +20,7 @@ function transformTap(fn, opt = {}) {
                 await fn(chunk, index++);
             }
             catch (err) {
-                console.error(err);
+                logger.error(err);
                 // suppressed error
             }
             cb(null, chunk); // pass through the item

package/dist/stream/transform/worker/workerClassProxy.js

@@ -10,6 +10,7 @@ if (!workerFile) {
 // console.log(`worker#${workerIndex} created`)
 
 try {
+  // require('esbuild-register') // alternative
   require('ts-node/register/transpile-only')
   require('tsconfig-paths/register')
 } catch {} // require if exists

package/dist/stream/writable/writableFork.d.ts

@@ -5,5 +5,7 @@ import { TransformOptions, WritableTyped } from '../stream.model';
  * Currently does NOT (!) maintain backpressure.
  * Error in the forked pipeline will propagate up to the main pipeline (and log error, to be sure).
  * Will wait until all forked pipelines are completed before completing the stream.
+ *
+ * @experimental
  */
 export declare function writableFork<T>(chains: NodeJS.WritableStream[][], opt?: TransformOptions): WritableTyped<T>;

package/dist/stream/writable/writableFork.js

@@ -8,6 +8,8 @@ const __1 = require("../..");
  * Currently does NOT (!) maintain backpressure.
  * Error in the forked pipeline will propagate up to the main pipeline (and log error, to be sure).
  * Will wait until all forked pipelines are completed before completing the stream.
+ *
+ * @experimental
  */
 function writableFork(chains, opt) {
     const readables = [];

package/dist/stream/writable/writableLimit.d.ts

@@ -0,0 +1,9 @@
+/// <reference types="node" />
+import { Readable } from 'stream';
+import { WritableTyped } from '../stream.model';
+/**
+ * Allows to stop the Readable stream after the pipeline has processed X number of rows.
+ * It counts OUTPUT rows (not input), because this Writable is always at the end of the Pipeline.
+ * It ensures that everything has been processed before issuing a STOP on the readable.
+ */
+export declare function writableLimit<T>(readable: Readable, limit: number): WritableTyped<T>;

package/dist/stream/writable/writableLimit.js

@@ -0,0 +1,29 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.writableLimit = void 0;
+const stream_1 = require("stream");
+/**
+ * Allows to stop the Readable stream after the pipeline has processed X number of rows.
+ * It counts OUTPUT rows (not input), because this Writable is always at the end of the Pipeline.
+ * It ensures that everything has been processed before issuing a STOP on the readable.
+ */
+function writableLimit(readable, limit) {
+    let i = 0;
+    return new stream_1.Writable({
+        objectMode: true,
+        write(chunk, _, cb) {
+            if (limit === 0)
+                return cb(); // no limit, just passthrough
+            i++;
+            if (i === limit) {
+                console.log(`writableLimit of ${limit} reached`);
+                readable.destroy();
+                cb(); // do we need it?
+            }
+            else {
+                cb(); // passthrough
+            }
+        },
+    });
+}
+exports.writableLimit = writableLimit;

package/dist/stream/writable/writableVoid.d.ts

@@ -1,9 +1,16 @@
 /// <reference types="node" />
 import { Writable } from 'stream';
+import { DeferredPromise } from '@naturalcycles/js-lib';
 import { TransformOptions } from '../stream.model';
+export interface WritableVoidOptions extends TransformOptions {
+    /**
+     * If set - it will be Resolved when the Stream is done (after final.cb)
+     */
+    streamDone?: DeferredPromise;
+}
 /**
  * Use as a "null-terminator" of stream.pipeline.
  * It consumes the stream as quickly as possible without doing anything.
  * Put it in the end of your pipeline in case it ends with Transform that needs a consumer.
  */
-export declare function writableVoid(opt?: TransformOptions): Writable;
+export declare function writableVoid(opt?: WritableVoidOptions): Writable;

package/dist/stream/writable/writableVoid.js

@@ -7,13 +7,17 @@ const stream_1 = require("stream");
  * It consumes the stream as quickly as possible without doing anything.
  * Put it in the end of your pipeline in case it ends with Transform that needs a consumer.
  */
-function writableVoid(opt) {
+function writableVoid(opt = {}) {
     return new stream_1.Writable({
         objectMode: true,
         ...opt,
         write(chunk, _encoding, cb) {
             cb();
         },
+        final(cb) {
+            cb();
+            opt.streamDone?.resolve();
+        },
     });
 }
 exports.writableVoid = writableVoid;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@naturalcycles/nodejs-lib",
-  "version": "12.56.0",
+  "version": "12.59.0",
   "scripts": {
     "prepare": "husky install",
     "docs-serve": "vuepress dev docs",