@naturalcycles/nodejs-lib 12.59.0 → 12.62.1

package/dist/stream/transform/transformLogProgress.js

@@ -7,6 +7,7 @@ const js_lib_1 = require("@naturalcycles/js-lib");
 const time_lib_1 = require("@naturalcycles/time-lib");
 const colors_1 = require("../../colors");
 const colors_2 = require("../../colors/colors");
+const sizeStack_1 = require("../sizeStack");
 const inspectOpt = {
     colors: colors_2.hasColors,
     breakLength: 300,
@@ -15,7 +16,7 @@ const inspectOpt = {
  * Pass-through transform that optionally logs progress.
  */
 function transformLogProgress(opt = {}) {
-    const { metric = 'progress', heapTotal: logHeapTotal = false, heapUsed: logHeapUsed = false, rss: logRss = true, peakRSS: logPeakRSS = true, logRPS = true, logEvery = 1000, batchSize = 1, extra, logger = console, } = opt;
+    const { metric = 'progress', heapTotal: logHeapTotal = false, heapUsed: logHeapUsed = false, rss: logRss = true, peakRSS: logPeakRSS = true, logRPS = true, logEvery = 1000, logSizes = false, logSizesBuffer = 100000, logZippedSizes = false, batchSize = 1, extra, logger = console, } = opt;
     const logProgress = opt.logProgress !== false && logEvery !== 0; // true by default
     const logEvery10 = logEvery * 10;
     const started = Date.now();
@@ -24,13 +25,19 @@ function transformLogProgress(opt = {}) {
     let processedLastSecond = 0;
     let progress = 0;
     let peakRSS = 0;
+    const sizes = logSizes ? new sizeStack_1.SizeStack('json', logSizesBuffer) : undefined;
+    const sizesZipped = logZippedSizes ? new sizeStack_1.SizeStack('json.gz', logSizesBuffer) : undefined;
     logStats(); // initial
     return new stream_1.Transform({
         objectMode: true,
         ...opt,
-        transform(chunk, _encoding, cb) {
+        transform(chunk, _, cb) {
             progress++;
             processedLastSecond++;
+            if (sizes) {
+                // Check it, cause gzipping might be delayed here..
+                void sizeStack_1.SizeStack.countItem(chunk, logger, sizes, sizesZipped);
+            }
             if (logProgress && progress % logEvery === 0) {
                 logStats(chunk, false, progress % logEvery10 === 0);
             }
@@ -54,23 +61,34 @@ function transformLogProgress(opt = {}) {
         const rps10 = Math.round(sma.push(lastRPS));
         if (mem.rss > peakRSS)
             peakRSS = mem.rss;
-        logger.log((0, util_1.inspect)({
+        const o = {
             [final ? `${metric}_final` : metric]: batchedProgress,
-            ...(extra ? extra(chunk, progress) : {}),
-            ...(logHeapUsed ? { heapUsed: (0, js_lib_1._mb)(mem.heapUsed) } : {}),
-            ...(logHeapTotal ? { heapTotal: (0, js_lib_1._mb)(mem.heapTotal) } : {}),
-            ...(logRss ? { rss: (0, js_lib_1._mb)(mem.rss) } : {}),
-            ...(logPeakRSS ? { peakRSS: (0, js_lib_1._mb)(peakRSS) } : {}),
-            ...(opt.rssMinusHeap ? { rssMinusHeap: (0, js_lib_1._mb)(mem.rss - mem.heapTotal) } : {}),
-            ...(opt.external ? { external: (0, js_lib_1._mb)(mem.external) } : {}),
-            ...(opt.arrayBuffers ? { arrayBuffers: (0, js_lib_1._mb)(mem.arrayBuffers || 0) } : {}),
-            ...(logRPS
-                ? {
-                    rps10,
-                    rpsTotal,
-                }
-                : {}),
-        }, inspectOpt));
+        };
+        if (extra)
+            Object.assign(o, extra(chunk, progress));
+        if (logHeapUsed)
+            o.heapUsed = (0, js_lib_1._mb)(mem.heapUsed);
+        if (logHeapTotal)
+            o.heapTotal = (0, js_lib_1._mb)(mem.heapTotal);
+        if (logRss)
+            o.rss = (0, js_lib_1._mb)(mem.rss);
+        if (logPeakRSS)
+            o.peakRSS = (0, js_lib_1._mb)(peakRSS);
+        if (opt.rssMinusHeap)
+            o.rssMinusHeap = (0, js_lib_1._mb)(mem.rss - mem.heapTotal);
+        if (opt.external)
+            o.external = (0, js_lib_1._mb)(mem.external);
+        if (opt.arrayBuffers)
+            o.arrayBuffers = (0, js_lib_1._mb)(mem.arrayBuffers || 0);
+        if (logRPS)
+            Object.assign(o, { rps10, rpsTotal });
+        logger.log((0, util_1.inspect)(o, inspectOpt));
+        if (sizes?.items.length) {
+            logger.log(sizes.getStats());
+            if (sizesZipped?.items.length) {
+                logger.log(sizesZipped.getStats());
+            }
+        }
         if (tenx) {
             let perHour = Math.round((batchedProgress * 1000 * 60 * 60) / (now - started)) || 0;
             if (perHour > 900) {

package/dist/stream/transform/transformMap.d.ts

@@ -11,9 +11,8 @@ export interface TransformMapOptions<IN = any, OUT = IN> {
      * Predicate to filter outgoing results (after mapper).
      * Allows to not emit all results.
      *
-     * Set to `r => r` (passthrough predicate) to pass ANY value (including undefined/null)
-     *
-     * @default to filter out undefined/null values, but pass anything else
+     * Defaults to "pass everything" (including null, undefined, etc).
+     * Simpler way to exclude certain cases is to return SKIP symbol from the mapper.
      */
     predicate?: AsyncPredicate<OUT>;
     /**
@@ -39,7 +38,6 @@ export interface TransformMapOptions<IN = any, OUT = IN> {
     metric?: string;
     logger?: CommonLogger;
 }
-export declare function notNullishPredicate(item: any): boolean;
 /**
  * Like pMap, but for streams.
  * Inspired by `through2`.

package/dist/stream/transform/transformMap.js

@@ -1,14 +1,10 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.transformMap = exports.notNullishPredicate = void 0;
+exports.transformMap = void 0;
 const js_lib_1 = require("@naturalcycles/js-lib");
 const through2Concurrent = require("through2-concurrent");
 const colors_1 = require("../../colors");
 const stream_util_1 = require("../stream.util");
-function notNullishPredicate(item) {
-    return item !== undefined && item !== null;
-}
-exports.notNullishPredicate = notNullishPredicate;
 // doesn't work, cause here we don't construct our Transform instance ourselves
 // export class TransformMap extends AbortableTransform {}
 /**
@@ -24,7 +20,8 @@ exports.notNullishPredicate = notNullishPredicate;
  * If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
  */
 function transformMap(mapper, opt = {}) {
-    const { concurrency = 16, predicate = notNullishPredicate, errorMode = js_lib_1.ErrorMode.THROW_IMMEDIATELY, flattenArrayOutput, onError, metric = 'stream', logger = console, } = opt;
+    const { concurrency = 16, predicate, // we now default to "no predicate" (meaning pass-everything)
+    errorMode = js_lib_1.ErrorMode.THROW_IMMEDIATELY, flattenArrayOutput, onError, metric = 'stream', logger = console, } = opt;
     let index = -1;
     let isSettled = false;
     let errors = 0;
@@ -44,24 +41,22 @@ function transformMap(mapper, opt = {}) {
             }
         },
     }, async function transformMapFn(chunk, _, cb) {
-        index++;
-        // console.log({chunk, _encoding})
         // Stop processing if isSettled (either THROW_IMMEDIATELY was fired or END received)
         if (isSettled)
             return cb();
+        const currentIndex = ++index;
         try {
-            const currentIndex = index; // because we need to pass it to 2 functions - mapper and predicate. Refers to INPUT index (since it may return multiple outputs)
             const res = await mapper(chunk, currentIndex);
             const passedResults = await (0, js_lib_1.pFilter)(flattenArrayOutput && Array.isArray(res) ? res : [res], async (r) => {
                 if (r === js_lib_1.END) {
                     isSettled = true; // will be checked later
                     return false;
                 }
-                return r !== js_lib_1.SKIP && (await predicate(r, currentIndex));
+                return r !== js_lib_1.SKIP && (!predicate || (await predicate(r, currentIndex)));
             });
             passedResults.forEach(r => this.push(r));
             if (isSettled) {
-                logger.log(`transformMap END received at index ${index}`);
+                logger.log(`transformMap END received at index ${currentIndex}`);
                 (0, stream_util_1.pipelineClose)('transformMap', this, this.sourceReadable, this.streamDone, logger);
             }
             cb(); // done processing

package/dist/stream/transform/transformMapSimple.js

@@ -17,7 +17,7 @@ function transformMapSimple(mapper, opt = {}) {
     const { errorMode = js_lib_1.ErrorMode.THROW_IMMEDIATELY, logger = console } = opt;
     return new stream_1.Transform({
         objectMode: true,
-        transform(chunk, _encoding, cb) {
+        transform(chunk, _, cb) {
             try {
                 cb(null, mapper(chunk, ++index));
             }

package/dist/stream/transform/transformMapSync.d.ts

@@ -1,4 +1,5 @@
 import { CommonLogger, ErrorMode, Mapper, Predicate } from '@naturalcycles/js-lib';
+import { AbortableTransform } from '../pipeline/pipeline';
 import { TransformTyped } from '../stream.model';
 export interface TransformMapSyncOptions<IN = any, OUT = IN> {
     /**
@@ -14,9 +15,8 @@ export interface TransformMapSyncOptions<IN = any, OUT = IN> {
      * Predicate to filter outgoing results (after mapper).
      * Allows to not emit all results.
      *
-     * @default to filter out undefined/null values, but pass anything else
-     *
-     * Set to `r => r` (passthrough predicate) to pass ANY value (including undefined/null)
+     * Defaults to "pass everything".
+     * Simpler way to skip individual entries is to return SKIP symbol.
      */
     predicate?: Predicate<OUT>;
     /**
@@ -36,6 +36,8 @@ export interface TransformMapSyncOptions<IN = any, OUT = IN> {
     metric?: string;
     logger?: CommonLogger;
 }
+export declare class TransformMapSync extends AbortableTransform {
+}
 /**
  * Sync (not async) version of transformMap.
  * Supposedly faster, for cases when async is not needed.

package/dist/stream/transform/transformMapSync.js

@@ -1,42 +1,48 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.transformMapSync = void 0;
-const stream_1 = require("stream");
+exports.transformMapSync = exports.TransformMapSync = void 0;
 const js_lib_1 = require("@naturalcycles/js-lib");
 const colors_1 = require("../../colors");
-const transformMap_1 = require("./transformMap");
+const pipeline_1 = require("../pipeline/pipeline");
+const stream_util_1 = require("../stream.util");
+class TransformMapSync extends pipeline_1.AbortableTransform {
+}
+exports.TransformMapSync = TransformMapSync;
 /**
  * Sync (not async) version of transformMap.
  * Supposedly faster, for cases when async is not needed.
  */
 function transformMapSync(mapper, opt = {}) {
     let index = -1;
-    const { predicate = transformMap_1.notNullishPredicate, errorMode = js_lib_1.ErrorMode.THROW_IMMEDIATELY, flattenArrayOutput = false, onError, metric = 'stream', objectMode = true, logger = console, } = opt;
-    let isRejected = false;
+    const { predicate, // defaults to "no predicate" (pass everything)
+    errorMode = js_lib_1.ErrorMode.THROW_IMMEDIATELY, flattenArrayOutput = false, onError, metric = 'stream', objectMode = true, logger = console, } = opt;
+    let isSettled = false;
     let errors = 0;
     const collectedErrors = []; // only used if errorMode == THROW_AGGREGATED
-    return new stream_1.Transform({
+    return new TransformMapSync({
         objectMode,
         ...opt,
-        transform(chunk, _encoding, cb) {
-            // Stop processing if THROW_IMMEDIATELY mode is used
-            if (isRejected && errorMode === js_lib_1.ErrorMode.THROW_IMMEDIATELY) {
+        transform(chunk, _, cb) {
+            // Stop processing if isSettled
+            if (isSettled)
                 return cb();
-            }
+            const currentIndex = ++index;
             try {
-                if (!predicate(chunk, ++index)) {
-                    cb(); // signal that we've finished processing, but emit no output here
-                    return;
-                }
                 // map and pass through
-                const v = mapper(chunk, index);
-                if (flattenArrayOutput && Array.isArray(v)) {
-                    // Pass each item individually
-                    v.forEach(item => this.push(item));
-                }
-                else {
-                    cb(null, v);
+                const v = mapper(chunk, currentIndex);
+                const passedResults = (flattenArrayOutput && Array.isArray(v) ? v : [v]).filter(r => {
+                    if (r === js_lib_1.END) {
+                        isSettled = true; // will be checked later
+                        return false;
+                    }
+                    return r !== js_lib_1.SKIP && (!predicate || predicate(r, currentIndex));
+                });
+                passedResults.forEach(r => this.push(r));
+                if (isSettled) {
+                    logger.log(`transformMapSync END received at index ${currentIndex}`);
+                    (0, stream_util_1.pipelineClose)('transformMapSync', this, this.sourceReadable, this.streamDone, logger);
                 }
+                cb(); // done processing
             }
             catch (err) {
                 logger.error(err);
@@ -49,7 +55,7 @@ function transformMapSync(mapper, opt = {}) {
                     catch { }
                 }
                 if (errorMode === js_lib_1.ErrorMode.THROW_IMMEDIATELY) {
-                    isRejected = true;
+                    isSettled = true;
                     // Emit error immediately
                     return cb(err);
                 }

package/dist/stream/transform/transformNoOp.js

@@ -10,7 +10,7 @@ const stream_1 = require("stream");
 function transformNoOp() {
     return new stream_1.Transform({
         objectMode: true,
-        transform(chunk, _encoding, cb) {
+        transform(chunk, _, cb) {
             cb(null, chunk);
         },
     });

package/dist/stream/transform/transformTap.js

@@ -10,14 +10,14 @@ const stream_1 = require("stream");
  */
 function transformTap(fn, opt = {}) {
     const { logger = console } = opt;
-    let index = 0;
+    let index = -1;
     return new stream_1.Transform({
         objectMode: true,
         ...opt,
-        async transform(chunk, _encoding, cb) {
+        async transform(chunk, _, cb) {
             // console.log('tap', chunk)
             try {
-                await fn(chunk, index++);
+                await fn(chunk, ++index);
             }
             catch (err) {
                 logger.error(err);

package/dist/stream/transform/transformToArray.js

@@ -10,7 +10,7 @@ function transformToArray(opt = {}) {
     return new stream_1.Transform({
         objectMode: true,
         ...opt,
-        transform(chunk, _encoding, cb) {
+        transform(chunk, _, cb) {
             res.push(chunk);
             // callback to signal that we processed input, but not emitting any output
             cb();

package/dist/stream/transform/transformToString.js

@@ -15,8 +15,8 @@ function transformToString() {
     return new stream_1.Transform({
         objectMode: false,
         readableObjectMode: true,
-        transform(chunk, _encoding, cb) {
-            // console.log(`enc: ${_encoding}`, chunk.toString())
+        transform(chunk, _, cb) {
+            // console.log(`enc: ${_}`, chunk.toString())
             cb(null, chunk.toString());
         },
     });

package/dist/stream/transform/worker/transformMultiThreaded.js

@@ -71,7 +71,7 @@ function transformMultiThreaded(opt) {
                 cb(err);
             }
         },
-    }, async function transformMapFn(chunk, _encoding, cb) {
+    }, async function transformMapFn(chunk, _, cb) {
         // Freezing the index, because it may change due to concurrency
         const currentIndex = ++index;
         // Create the unresolved promise (to avait)

package/dist/stream/writable/writableFork.js

@@ -24,7 +24,7 @@ function writableFork(chains, opt) {
     return new stream_1.Writable({
         objectMode: true,
         ...opt,
-        write(chunk, _encoding, cb) {
+        write(chunk, _, cb) {
             // Push/fork to all sub-streams
             // No backpressure is ensured here, it'll push regardless of the
             readables.forEach(readable => readable.push(chunk));

package/dist/stream/writable/writablePushToArray.js

@@ -9,7 +9,7 @@ function writablePushToArray(arr, opt = {}) {
     return new stream_1.Writable({
         objectMode: true,
         ...opt,
-        write(chunk, _encoding, cb) {
+        write(chunk, _, cb) {
             arr.push(chunk);
             // callback to signal that we processed input, but not emitting any output
             cb();

package/dist/stream/writable/writableVoid.js

@@ -11,7 +11,7 @@ function writableVoid(opt = {}) {
     return new stream_1.Writable({
         objectMode: true,
         ...opt,
-        write(chunk, _encoding, cb) {
+        write(chunk, _, cb) {
             cb();
         },
         final(cb) {

package/dist/util/zip.util.d.ts

@@ -1,18 +1,26 @@
 /// <reference types="node" />
 import { ZlibOptions } from 'zlib';
 /**
- * zipBuffer uses `deflate`.
- * It's 9 bytes shorter than gzip.
+ * deflateBuffer uses `deflate`.
+ * It's 9 bytes shorter than `gzip`.
  */
-export declare function zipBuffer(buf: Buffer, options?: ZlibOptions): Promise<Buffer>;
+export declare function deflateBuffer(buf: Buffer, options?: ZlibOptions): Promise<Buffer>;
+export declare function inflateBuffer(buf: Buffer, options?: ZlibOptions): Promise<Buffer>;
+/**
+ * deflateString uses `deflate`.
+ * It's 9 bytes shorter than `gzip`.
+ */
+export declare function deflateString(s: string, options?: ZlibOptions): Promise<Buffer>;
+export declare function inflateToString(buf: Buffer, options?: ZlibOptions): Promise<string>;
 /**
  * gzipBuffer uses `gzip`
- * It's 9 bytes longer than deflate.
+ * It's 9 bytes longer than `deflate`.
  */
 export declare function gzipBuffer(buf: Buffer, options?: ZlibOptions): Promise<Buffer>;
-export declare function unzipBuffer(buf: Buffer, options?: ZlibOptions): Promise<Buffer>;
 export declare function gunzipBuffer(buf: Buffer, options?: ZlibOptions): Promise<Buffer>;
-export declare function zipString(s: string, options?: ZlibOptions): Promise<Buffer>;
+/**
+ * gzipString uses `gzip`.
+ * It's 9 bytes longer than `deflate`.
+ */
 export declare function gzipString(s: string, options?: ZlibOptions): Promise<Buffer>;
-export declare function unzipToString(buf: Buffer, options?: ZlibOptions): Promise<string>;
 export declare function gunzipToString(buf: Buffer, options?: ZlibOptions): Promise<string>;

package/dist/util/zip.util.js

@@ -1,52 +1,57 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.gunzipToString = exports.unzipToString = exports.gzipString = exports.zipString = exports.gunzipBuffer = exports.unzipBuffer = exports.gzipBuffer = exports.zipBuffer = void 0;
+exports.gunzipToString = exports.gzipString = exports.gunzipBuffer = exports.gzipBuffer = exports.inflateToString = exports.deflateString = exports.inflateBuffer = exports.deflateBuffer = void 0;
 const util_1 = require("util");
 const zlib = require("zlib");
 const deflate = (0, util_1.promisify)(zlib.deflate.bind(zlib));
 const inflate = (0, util_1.promisify)(zlib.inflate.bind(zlib));
 const gzip = (0, util_1.promisify)(zlib.gzip.bind(zlib));
 const gunzip = (0, util_1.promisify)(zlib.gunzip.bind(zlib));
-// string > zip
+// string > compressed buffer
 /**
- * zipBuffer uses `deflate`.
- * It's 9 bytes shorter than gzip.
+ * deflateBuffer uses `deflate`.
+ * It's 9 bytes shorter than `gzip`.
  */
-async function zipBuffer(buf, options = {}) {
+async function deflateBuffer(buf, options = {}) {
     return await deflate(buf, options);
 }
-exports.zipBuffer = zipBuffer;
+exports.deflateBuffer = deflateBuffer;
+async function inflateBuffer(buf, options = {}) {
+    return await inflate(buf, options);
+}
+exports.inflateBuffer = inflateBuffer;
+/**
+ * deflateString uses `deflate`.
+ * It's 9 bytes shorter than `gzip`.
+ */
+async function deflateString(s, options) {
+    return await deflateBuffer(Buffer.from(s), options);
+}
+exports.deflateString = deflateString;
+async function inflateToString(buf, options) {
+    return (await inflateBuffer(buf, options)).toString();
+}
+exports.inflateToString = inflateToString;
 /**
  * gzipBuffer uses `gzip`
- * It's 9 bytes longer than deflate.
+ * It's 9 bytes longer than `deflate`.
  */
 async function gzipBuffer(buf, options = {}) {
     return await gzip(buf, options);
 }
 exports.gzipBuffer = gzipBuffer;
-// zip > buffer
-async function unzipBuffer(buf, options = {}) {
-    return await inflate(buf, options);
-}
-exports.unzipBuffer = unzipBuffer;
 async function gunzipBuffer(buf, options = {}) {
     return await gunzip(buf, options);
 }
 exports.gunzipBuffer = gunzipBuffer;
-// convenience
-async function zipString(s, options) {
-    return await zipBuffer(Buffer.from(s), options);
-}
-exports.zipString = zipString;
+/**
+ * gzipString uses `gzip`.
+ * It's 9 bytes longer than `deflate`.
+ */
 async function gzipString(s, options) {
     return await gzipBuffer(Buffer.from(s), options);
 }
 exports.gzipString = gzipString;
-// convenience
-async function unzipToString(buf, options) {
-    return (await unzipBuffer(buf, options)).toString();
-}
-exports.unzipToString = unzipToString;
 async function gunzipToString(buf, options) {
     return (await gunzipBuffer(buf, options)).toString();
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@naturalcycles/nodejs-lib",
-  "version": "12.59.0",
+  "version": "12.62.1",
   "scripts": {
     "prepare": "husky install",
     "docs-serve": "vuepress dev docs",
@@ -32,7 +32,7 @@
     "fs-extra": "^10.0.0",
     "globby": "^11.0.0",
     "got": "^11.0.1",
-    "joi": "^17.2.1",
+    "joi": "17.4.2",
     "lru-cache": "^6.0.0",
     "move-file": "^2.0.0",
     "nanoid": "^3.0.0",