@naturalcycles/nodejs-lib 12.56.1 → 12.60.0

package/dist/stream/transform/transformMapSimple.js

@@ -14,15 +14,15 @@ const js_lib_1 = require("@naturalcycles/js-lib");
  */
 function transformMapSimple(mapper, opt = {}) {
     let index = -1;
-    const { errorMode = js_lib_1.ErrorMode.THROW_IMMEDIATELY } = opt;
+    const { errorMode = js_lib_1.ErrorMode.THROW_IMMEDIATELY, logger = console } = opt;
     return new stream_1.Transform({
         objectMode: true,
-        transform(chunk, _encoding, cb) {
+        transform(chunk, _, cb) {
             try {
                 cb(null, mapper(chunk, ++index));
             }
             catch (err) {
-                console.error(err);
+                logger.error(err);
                 if (errorMode === js_lib_1.ErrorMode.SUPPRESS) {
                     cb(); // suppress the error
                 }

package/dist/stream/transform/transformMapSync.d.ts

@@ -1,4 +1,5 @@
-import { ErrorMode, Mapper, Predicate } from '@naturalcycles/js-lib';
+import { CommonLogger, ErrorMode, Mapper, Predicate } from '@naturalcycles/js-lib';
+import { AbortableTransform } from '../pipeline/pipeline';
 import { TransformTyped } from '../stream.model';
 export interface TransformMapSyncOptions<IN = any, OUT = IN> {
     /**
@@ -14,9 +15,8 @@ export interface TransformMapSyncOptions<IN = any, OUT = IN> {
      * Predicate to filter outgoing results (after mapper).
      * Allows to not emit all results.
      *
-     * @default to filter out undefined/null values, but pass anything else
-     *
-     * Set to `r => r` (passthrough predicate) to pass ANY value (including undefined/null)
+     * Defaults to "pass everything".
+     * Simpler way to skip individual entries is to return SKIP symbol.
      */
     predicate?: Predicate<OUT>;
     /**
@@ -34,6 +34,9 @@ export interface TransformMapSyncOptions<IN = any, OUT = IN> {
      * @default `stream`
      */
     metric?: string;
+    logger?: CommonLogger;
+}
+export declare class TransformMapSync extends AbortableTransform {
 }
 /**
  * Sync (not async) version of transformMap.

package/dist/stream/transform/transformMapSync.js

@@ -1,45 +1,51 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.transformMapSync = void 0;
-const stream_1 = require("stream");
+exports.transformMapSync = exports.TransformMapSync = void 0;
 const js_lib_1 = require("@naturalcycles/js-lib");
 const colors_1 = require("../../colors");
-const transformMap_1 = require("./transformMap");
+const pipeline_1 = require("../pipeline/pipeline");
+const stream_util_1 = require("../stream.util");
+class TransformMapSync extends pipeline_1.AbortableTransform {
+}
+exports.TransformMapSync = TransformMapSync;
 /**
  * Sync (not async) version of transformMap.
  * Supposedly faster, for cases when async is not needed.
  */
 function transformMapSync(mapper, opt = {}) {
     let index = -1;
-    const { predicate = transformMap_1.notNullishPredicate, errorMode = js_lib_1.ErrorMode.THROW_IMMEDIATELY, flattenArrayOutput = false, onError, metric = 'stream', objectMode = true, } = opt;
-    let isRejected = false;
+    const { predicate, // defaults to "no predicate" (pass everything)
+    errorMode = js_lib_1.ErrorMode.THROW_IMMEDIATELY, flattenArrayOutput = false, onError, metric = 'stream', objectMode = true, logger = console, } = opt;
+    let isSettled = false;
     let errors = 0;
     const collectedErrors = []; // only used if errorMode == THROW_AGGREGATED
-    return new stream_1.Transform({
+    return new TransformMapSync({
         objectMode,
         ...opt,
-        transform(chunk, _encoding, cb) {
-            // Stop processing if THROW_IMMEDIATELY mode is used
-            if (isRejected && errorMode === js_lib_1.ErrorMode.THROW_IMMEDIATELY) {
+        transform(chunk, _, cb) {
+            // Stop processing if isSettled
+            if (isSettled)
                 return cb();
-            }
+            const currentIndex = ++index;
             try {
-                if (!predicate(chunk, ++index)) {
-                    cb(); // signal that we've finished processing, but emit no output here
-                    return;
-                }
                 // map and pass through
-                const v = mapper(chunk, index);
-                if (flattenArrayOutput && Array.isArray(v)) {
-                    // Pass each item individually
-                    v.forEach(item => this.push(item));
-                }
-                else {
-                    cb(null, v);
+                const v = mapper(chunk, currentIndex);
+                const passedResults = (flattenArrayOutput && Array.isArray(v) ? v : [v]).filter(r => {
+                    if (r === js_lib_1.END) {
+                        isSettled = true; // will be checked later
+                        return false;
+                    }
+                    return r !== js_lib_1.SKIP && (!predicate || predicate(r, currentIndex));
+                });
+                passedResults.forEach(r => this.push(r));
+                if (isSettled) {
+                    logger.log(`transformMapSync END received at index ${currentIndex}`);
+                    (0, stream_util_1.pipelineClose)('transformMapSync', this, this.sourceReadable, this.streamDone, logger);
                 }
+                cb(); // done processing
             }
             catch (err) {
-                console.error(err);
+                logger.error(err);
                 errors++;
                 logErrorStats();
                 if (onError) {
@@ -49,7 +55,7 @@ function transformMapSync(mapper, opt = {}) {
                     catch { }
                 }
                 if (errorMode === js_lib_1.ErrorMode.THROW_IMMEDIATELY) {
-                    isRejected = true;
+                    isSettled = true;
                     // Emit error immediately
                     return cb(err);
                 }
@@ -75,7 +81,7 @@ function transformMapSync(mapper, opt = {}) {
     function logErrorStats(final = false) {
         if (!errors)
             return;
-        console.log(`${metric} ${final ? 'final ' : ''}errors: ${(0, colors_1.yellow)(errors)}`);
+        logger.log(`${metric} ${final ? 'final ' : ''}errors: ${(0, colors_1.yellow)(errors)}`);
     }
 }
 exports.transformMapSync = transformMapSync;

package/dist/stream/transform/transformNoOp.js

@@ -10,7 +10,7 @@ const stream_1 = require("stream");
 function transformNoOp() {
     return new stream_1.Transform({
         objectMode: true,
-        transform(chunk, _encoding, cb) {
+        transform(chunk, _, cb) {
             cb(null, chunk);
         },
     });

package/dist/stream/transform/transformTap.d.ts

@@ -1,9 +1,12 @@
-import { AsyncMapper } from '@naturalcycles/js-lib';
+import { AsyncMapper, CommonLogger } from '@naturalcycles/js-lib';
 import { TransformOptions, TransformTyped } from '../stream.model';
+export interface TransformTapOptions extends TransformOptions {
+    logger?: CommonLogger;
+}
 /**
  * Similar to RxJS `tap` - allows to run a function for each stream item, without affecting the result.
  * Item is passed through to the output.
  *
  * Can also act as a counter, since `index` is passed to `fn`
  */
-export declare function transformTap<IN>(fn: AsyncMapper<IN, any>, opt?: TransformOptions): TransformTyped<IN, IN>;
+export declare function transformTap<IN>(fn: AsyncMapper<IN, any>, opt?: TransformTapOptions): TransformTyped<IN, IN>;

package/dist/stream/transform/transformTap.js

@@ -9,17 +9,18 @@ const stream_1 = require("stream");
  * Can also act as a counter, since `index` is passed to `fn`
  */
 function transformTap(fn, opt = {}) {
-    let index = 0;
+    const { logger = console } = opt;
+    let index = -1;
     return new stream_1.Transform({
         objectMode: true,
         ...opt,
-        async transform(chunk, _encoding, cb) {
+        async transform(chunk, _, cb) {
             // console.log('tap', chunk)
             try {
-                await fn(chunk, index++);
+                await fn(chunk, ++index);
             }
             catch (err) {
-                console.error(err);
+                logger.error(err);
                 // suppressed error
             }
             cb(null, chunk); // pass through the item

package/dist/stream/transform/transformToArray.js

@@ -10,7 +10,7 @@ function transformToArray(opt = {}) {
     return new stream_1.Transform({
         objectMode: true,
         ...opt,
-        transform(chunk, _encoding, cb) {
+        transform(chunk, _, cb) {
             res.push(chunk);
             // callback to signal that we processed input, but not emitting any output
             cb();

package/dist/stream/transform/transformToString.js

@@ -15,8 +15,8 @@ function transformToString() {
     return new stream_1.Transform({
         objectMode: false,
         readableObjectMode: true,
-        transform(chunk, _encoding, cb) {
-            // console.log(`enc: ${_encoding}`, chunk.toString())
+        transform(chunk, _, cb) {
+            // console.log(`enc: ${_}`, chunk.toString())
             cb(null, chunk.toString());
         },
     });

package/dist/stream/transform/worker/transformMultiThreaded.js

@@ -71,7 +71,7 @@ function transformMultiThreaded(opt) {
                 cb(err);
             }
         },
-    }, async function transformMapFn(chunk, _encoding, cb) {
+    }, async function transformMapFn(chunk, _, cb) {
         // Freezing the index, because it may change due to concurrency
         const currentIndex = ++index;
         // Create the unresolved promise (to avait)

package/dist/stream/transform/worker/workerClassProxy.js

@@ -10,6 +10,7 @@ if (!workerFile) {
 // console.log(`worker#${workerIndex} created`)
 
 try {
+  // require('esbuild-register') // alternative
   require('ts-node/register/transpile-only')
   require('tsconfig-paths/register')
 } catch {} // require if exists

package/dist/stream/writable/writableFork.d.ts

@@ -5,5 +5,7 @@ import { TransformOptions, WritableTyped } from '../stream.model';
  * Currently does NOT (!) maintain backpressure.
  * Error in the forked pipeline will propagate up to the main pipeline (and log error, to be sure).
  * Will wait until all forked pipelines are completed before completing the stream.
+ *
+ * @experimental
  */
 export declare function writableFork<T>(chains: NodeJS.WritableStream[][], opt?: TransformOptions): WritableTyped<T>;

package/dist/stream/writable/writableFork.js

@@ -8,6 +8,8 @@ const __1 = require("../..");
  * Currently does NOT (!) maintain backpressure.
  * Error in the forked pipeline will propagate up to the main pipeline (and log error, to be sure).
  * Will wait until all forked pipelines are completed before completing the stream.
+ *
+ * @experimental
  */
 function writableFork(chains, opt) {
     const readables = [];
@@ -22,7 +24,7 @@ function writableFork(chains, opt) {
     return new stream_1.Writable({
         objectMode: true,
         ...opt,
-        write(chunk, _encoding, cb) {
+        write(chunk, _, cb) {
             // Push/fork to all sub-streams
             // No backpressure is ensured here, it'll push regardless of the
             readables.forEach(readable => readable.push(chunk));

package/dist/stream/writable/writableLimit.d.ts

@@ -0,0 +1,9 @@
+/// <reference types="node" />
+import { Readable } from 'stream';
+import { WritableTyped } from '../stream.model';
+/**
+ * Allows to stop the Readable stream after the pipeline has processed X number of rows.
+ * It counts OUTPUT rows (not input), because this Writable is always at the end of the Pipeline.
+ * It ensures that everything has been processed before issuing a STOP on the readable.
+ */
+export declare function writableLimit<T>(readable: Readable, limit: number): WritableTyped<T>;

package/dist/stream/writable/writableLimit.js

@@ -0,0 +1,29 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.writableLimit = void 0;
+const stream_1 = require("stream");
+/**
+ * Allows to stop the Readable stream after the pipeline has processed X number of rows.
+ * It counts OUTPUT rows (not input), because this Writable is always at the end of the Pipeline.
+ * It ensures that everything has been processed before issuing a STOP on the readable.
+ */
+function writableLimit(readable, limit) {
+    let i = 0;
+    return new stream_1.Writable({
+        objectMode: true,
+        write(chunk, _, cb) {
+            if (limit === 0)
+                return cb(); // no limit, just passthrough
+            i++;
+            if (i === limit) {
+                console.log(`writableLimit of ${limit} reached`);
+                readable.destroy();
+                cb(); // do we need it?
+            }
+            else {
+                cb(); // passthrough
+            }
+        },
+    });
+}
+exports.writableLimit = writableLimit;

package/dist/stream/writable/writablePushToArray.js

@@ -9,7 +9,7 @@ function writablePushToArray(arr, opt = {}) {
     return new stream_1.Writable({
         objectMode: true,
         ...opt,
-        write(chunk, _encoding, cb) {
+        write(chunk, _, cb) {
             arr.push(chunk);
             // callback to signal that we processed input, but not emitting any output
             cb();

package/dist/stream/writable/writableVoid.d.ts

@@ -1,9 +1,16 @@
 /// <reference types="node" />
 import { Writable } from 'stream';
+import { DeferredPromise } from '@naturalcycles/js-lib';
 import { TransformOptions } from '../stream.model';
+export interface WritableVoidOptions extends TransformOptions {
+    /**
+     * If set - it will be Resolved when the Stream is done (after final.cb)
+     */
+    streamDone?: DeferredPromise;
+}
 /**
  * Use as a "null-terminator" of stream.pipeline.
  * It consumes the stream as quickly as possible without doing anything.
  * Put it in the end of your pipeline in case it ends with Transform that needs a consumer.
  */
-export declare function writableVoid(opt?: TransformOptions): Writable;
+export declare function writableVoid(opt?: WritableVoidOptions): Writable;

package/dist/stream/writable/writableVoid.js

@@ -7,13 +7,17 @@ const stream_1 = require("stream");
  * It consumes the stream as quickly as possible without doing anything.
  * Put it in the end of your pipeline in case it ends with Transform that needs a consumer.
  */
-function writableVoid(opt) {
+function writableVoid(opt = {}) {
     return new stream_1.Writable({
         objectMode: true,
         ...opt,
-        write(chunk, _encoding, cb) {
+        write(chunk, _, cb) {
             cb();
         },
+        final(cb) {
+            cb();
+            opt.streamDone?.resolve();
+        },
     });
 }
 exports.writableVoid = writableVoid;

package/dist/util/zip.util.d.ts

@@ -2,17 +2,25 @@
 import { ZlibOptions } from 'zlib';
 /**
  * zipBuffer uses `deflate`.
- * It's 9 bytes shorter than gzip.
+ * It's 9 bytes shorter than `gzip`.
  */
 export declare function zipBuffer(buf: Buffer, options?: ZlibOptions): Promise<Buffer>;
 /**
  * gzipBuffer uses `gzip`
- * It's 9 bytes longer than deflate.
+ * It's 9 bytes longer than `deflate`.
  */
 export declare function gzipBuffer(buf: Buffer, options?: ZlibOptions): Promise<Buffer>;
 export declare function unzipBuffer(buf: Buffer, options?: ZlibOptions): Promise<Buffer>;
 export declare function gunzipBuffer(buf: Buffer, options?: ZlibOptions): Promise<Buffer>;
+/**
+ * zipString uses `deflate`.
+ * It's 9 bytes shorter than `gzip`.
+ */
 export declare function zipString(s: string, options?: ZlibOptions): Promise<Buffer>;
+/**
+ * gzipString uses `gzip`.
+ * It's 9 bytes longer than `deflate`.
+ */
 export declare function gzipString(s: string, options?: ZlibOptions): Promise<Buffer>;
 export declare function unzipToString(buf: Buffer, options?: ZlibOptions): Promise<string>;
 export declare function gunzipToString(buf: Buffer, options?: ZlibOptions): Promise<string>;

package/dist/util/zip.util.js

@@ -10,7 +10,7 @@ const gunzip = (0, util_1.promisify)(zlib.gunzip.bind(zlib));
 // string > zip
 /**
  * zipBuffer uses `deflate`.
- * It's 9 bytes shorter than gzip.
+ * It's 9 bytes shorter than `gzip`.
  */
 async function zipBuffer(buf, options = {}) {
     return await deflate(buf, options);
@@ -18,7 +18,7 @@ async function zipBuffer(buf, options = {}) {
 exports.zipBuffer = zipBuffer;
 /**
  * gzipBuffer uses `gzip`
- * It's 9 bytes longer than deflate.
+ * It's 9 bytes longer than `deflate`.
  */
 async function gzipBuffer(buf, options = {}) {
     return await gzip(buf, options);
@@ -33,11 +33,18 @@ async function gunzipBuffer(buf, options = {}) {
     return await gunzip(buf, options);
 }
 exports.gunzipBuffer = gunzipBuffer;
-// convenience
+/**
+ * zipString uses `deflate`.
+ * It's 9 bytes shorter than `gzip`.
+ */
 async function zipString(s, options) {
     return await zipBuffer(Buffer.from(s), options);
 }
 exports.zipString = zipString;
+/**
+ * gzipString uses `gzip`.
+ * It's 9 bytes longer than `deflate`.
+ */
 async function gzipString(s, options) {
     return await gzipBuffer(Buffer.from(s), options);
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@naturalcycles/nodejs-lib",
-  "version": "12.56.1",
+  "version": "12.60.0",
   "scripts": {
     "prepare": "husky install",
     "docs-serve": "vuepress dev docs",