@naturalcycles/nodejs-lib 12.55.1 → 12.58.0

package/dist/index.d.ts

@@ -47,7 +47,7 @@ import { transformToString } from './stream/transform/transformToString';
 import { BaseWorkerClass, WorkerClassInterface } from './stream/transform/worker/baseWorkerClass';
 import { transformMultiThreaded, TransformMultiThreadedOptions } from './stream/transform/worker/transformMultiThreaded';
 import { WorkerInput, WorkerOutput } from './stream/transform/worker/transformMultiThreaded.model';
-import { writableForEach } from './stream/writable/writableForEach';
+export * from './stream/writable/writableForEach';
 import { writableFork } from './stream/writable/writableFork';
 import { writablePushToArray } from './stream/writable/writablePushToArray';
 import { writableVoid } from './stream/writable/writableVoid';
@@ -67,4 +67,4 @@ import { convert, getValidationResult, isValid, JoiValidationResult, undefinedIf
 import { sanitizeHTML, SanitizeHTMLOptions } from './validation/sanitize.util';
 import { runScript, RunScriptOptions } from './script';
 export type { RunScriptOptions, JoiValidationErrorData, JoiValidationResult, ValidationErrorItem, ExtendedJoi, SchemaTyped, AnySchema, AnySchemaTyped, ArraySchemaTyped, BooleanSchemaTyped, NumberSchemaTyped, ObjectSchemaTyped, StringSchemaTyped, IDebug, IDebugger, SlackServiceCfg, SlackMessage, SlackMessageProps, SlackApiBody, SlackMessagePrefixHook, ReadableTyped, WritableTyped, TransformTyped, PipelineFromNDJsonFileOptions, PipelineToNDJsonFileOptions, TransformJsonParseOptions, TransformToNDJsonOptions, TransformMapOptions, TransformMapSyncOptions, NDJSONStreamForEachOptions, TransformOptions, TransformLogProgressOptions, TransformMultiThreadedOptions, WorkerClassInterface, WorkerInput, WorkerOutput, TableDiffOptions, InspectAnyOptions, Got, GetGotOptions, AfterResponseHook, BeforeErrorHook, BeforeRequestHook, AjvValidationOptions, AjvSchemaCfg, AjvValidationErrorData, SanitizeHTMLOptions, };
-export { JoiValidationError, validate, getValidationResult, isValid, undefinedIfInvalid, convert, Joi, booleanSchema, booleanDefaultToFalseSchema, stringSchema, numberSchema, integerSchema, percentageSchema, dateStringSchema, arraySchema, binarySchema, objectSchema, oneOfSchema, anySchema, anyObjectSchema, baseDBEntitySchema, savedDBEntitySchema, idSchema, unixTimestampSchema, verSchema, emailSchema, SEM_VER_PATTERN, semVerSchema, userAgentSchema, utcOffsetSchema, ipAddressSchema, slugSchema, urlSchema, processSharedUtil, zipBuffer, gzipBuffer, unzipBuffer, gunzipBuffer, zipString, gzipString, unzipToString, gunzipToString, requireEnvKeys, requireFileToExist, LRUMemoCache, stringId, stringIdAsync, stringIdUnsafe, ALPHABET_NUMBER, ALPHABET_LOWERCASE, ALPHABET_UPPERCASE, ALPHABET_ALPHANUMERIC_LOWERCASE, ALPHABET_ALPHANUMERIC_UPPERCASE, ALPHABET_ALPHANUMERIC, md5, hash, hashAsBuffer, md5AsBuffer, stringToBase64, base64ToString, bufferToBase64, base64ToBuffer, Debug, getSecretMap, setSecretMap, loadSecretsFromEnv, loadSecretsFromJsonFile, removeSecretsFromEnv, secret, secretOptional, memoryUsage, memoryUsageFull, SlackService, slackDefaultMessagePrefixHook, readableCreate, readableFrom, readableFromArray, readableToArray, readableForEach, readableForEachSync, readableMap, readableMapToArray, _pipeline, transformBuffer, ndjsonMap, ndJsonFileRead, ndJsonFileWrite, ndjsonStreamForEach, pipelineFromNDJsonFile, pipelineToNDJsonFile, NDJsonStats, streamToNDJsonFile, transformJsonParse, bufferReviver, transformToNDJson, transformFilter, transformFilterSync, transformMap, transformMapSync, transformMapSimple, transformNoOp, writableForEach, writablePushToArray, transformSplit, transformToString, transformToArray, transformTap, transformLogProgress, transformLimit, writableVoid, writableFork, transformMultiThreaded, BaseWorkerClass, tableDiff, inspectAny, inspectAnyStringifyFn, getGot, HTTPError, TimeoutError, _chunkBuffer, Ajv, getAjv, AjvSchema, AjvValidationError, readJsonSchemas, readAjvSchemas, hasColors, sanitizeHTML, runScript, };
+export { JoiValidationError, validate, getValidationResult, isValid, undefinedIfInvalid, convert, Joi, booleanSchema, booleanDefaultToFalseSchema, stringSchema, numberSchema, integerSchema, percentageSchema, dateStringSchema, arraySchema, binarySchema, objectSchema, oneOfSchema, anySchema, anyObjectSchema, baseDBEntitySchema, savedDBEntitySchema, idSchema, unixTimestampSchema, verSchema, emailSchema, SEM_VER_PATTERN, semVerSchema, userAgentSchema, utcOffsetSchema, ipAddressSchema, slugSchema, urlSchema, processSharedUtil, zipBuffer, gzipBuffer, unzipBuffer, gunzipBuffer, zipString, gzipString, unzipToString, gunzipToString, requireEnvKeys, requireFileToExist, LRUMemoCache, stringId, stringIdAsync, stringIdUnsafe, ALPHABET_NUMBER, ALPHABET_LOWERCASE, ALPHABET_UPPERCASE, ALPHABET_ALPHANUMERIC_LOWERCASE, ALPHABET_ALPHANUMERIC_UPPERCASE, ALPHABET_ALPHANUMERIC, md5, hash, hashAsBuffer, md5AsBuffer, stringToBase64, base64ToString, bufferToBase64, base64ToBuffer, Debug, getSecretMap, setSecretMap, loadSecretsFromEnv, loadSecretsFromJsonFile, removeSecretsFromEnv, secret, secretOptional, memoryUsage, memoryUsageFull, SlackService, slackDefaultMessagePrefixHook, readableCreate, readableFrom, readableFromArray, readableToArray, readableForEach, readableForEachSync, readableMap, readableMapToArray, _pipeline, transformBuffer, ndjsonMap, ndJsonFileRead, ndJsonFileWrite, ndjsonStreamForEach, pipelineFromNDJsonFile, pipelineToNDJsonFile, NDJsonStats, streamToNDJsonFile, transformJsonParse, bufferReviver, transformToNDJson, transformFilter, transformFilterSync, transformMap, transformMapSync, transformMapSimple, transformNoOp, writablePushToArray, transformSplit, transformToString, transformToArray, transformTap, transformLogProgress, transformLimit, writableVoid, writableFork, transformMultiThreaded, BaseWorkerClass, tableDiff, inspectAny, inspectAnyStringifyFn, getGot, HTTPError, TimeoutError, _chunkBuffer, Ajv, getAjv, AjvSchema, AjvValidationError, readJsonSchemas, readAjvSchemas, hasColors, sanitizeHTML, runScript, };

package/dist/index.js

@@ -2,7 +2,8 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ALPHABET_LOWERCASE = exports.ALPHABET_NUMBER = exports.stringIdUnsafe = exports.stringIdAsync = exports.stringId = exports.LRUMemoCache = exports.requireFileToExist = exports.requireEnvKeys = exports.gunzipToString = exports.unzipToString = exports.gzipString = exports.zipString = exports.gunzipBuffer = exports.unzipBuffer = exports.gzipBuffer = exports.zipBuffer = exports.processSharedUtil = exports.urlSchema = exports.slugSchema = exports.ipAddressSchema = exports.utcOffsetSchema = exports.userAgentSchema = exports.semVerSchema = exports.SEM_VER_PATTERN = exports.emailSchema = exports.verSchema = exports.unixTimestampSchema = exports.idSchema = exports.savedDBEntitySchema = exports.baseDBEntitySchema = exports.anyObjectSchema = exports.anySchema = exports.oneOfSchema = exports.objectSchema = exports.binarySchema = exports.arraySchema = exports.dateStringSchema = exports.percentageSchema = exports.integerSchema = exports.numberSchema = exports.stringSchema = exports.booleanDefaultToFalseSchema = exports.booleanSchema = exports.Joi = exports.convert = exports.undefinedIfInvalid = exports.isValid = exports.getValidationResult = exports.validate = exports.JoiValidationError = void 0;
 exports.transformMapSimple = exports.transformMapSync = exports.transformMap = exports.transformFilterSync = exports.transformFilter = exports.transformToNDJson = exports.bufferReviver = exports.transformJsonParse = exports.streamToNDJsonFile = exports.NDJsonStats = exports.pipelineToNDJsonFile = exports.pipelineFromNDJsonFile = exports.ndjsonStreamForEach = exports.ndJsonFileWrite = exports.ndJsonFileRead = exports.ndjsonMap = exports.transformBuffer = exports._pipeline = exports.readableMapToArray = exports.readableMap = exports.readableForEachSync = exports.readableForEach = exports.readableToArray = exports.readableFromArray = exports.readableFrom = exports.readableCreate = exports.slackDefaultMessagePrefixHook = exports.SlackService = exports.memoryUsageFull = exports.memoryUsage = exports.secretOptional = exports.secret = exports.removeSecretsFromEnv = exports.loadSecretsFromJsonFile = exports.loadSecretsFromEnv = exports.setSecretMap = exports.getSecretMap = exports.Debug = exports.base64ToBuffer = exports.bufferToBase64 = exports.base64ToString = exports.stringToBase64 = exports.md5AsBuffer = exports.hashAsBuffer = exports.hash = exports.md5 = exports.ALPHABET_ALPHANUMERIC = exports.ALPHABET_ALPHANUMERIC_UPPERCASE = exports.ALPHABET_ALPHANUMERIC_LOWERCASE = exports.ALPHABET_UPPERCASE = void 0;
-exports.runScript = exports.sanitizeHTML = exports.hasColors = exports.readAjvSchemas = exports.readJsonSchemas = exports.AjvValidationError = exports.AjvSchema = exports.getAjv = exports.Ajv = exports._chunkBuffer = exports.TimeoutError = exports.HTTPError = exports.getGot = exports.inspectAnyStringifyFn = exports.inspectAny = exports.tableDiff = exports.BaseWorkerClass = exports.transformMultiThreaded = exports.writableFork = exports.writableVoid = exports.transformLimit = exports.transformLogProgress = exports.transformTap = exports.transformToArray = exports.transformToString = exports.transformSplit = exports.writablePushToArray = exports.writableForEach = exports.transformNoOp = void 0;
+exports.runScript = exports.sanitizeHTML = exports.hasColors = exports.readAjvSchemas = exports.readJsonSchemas = exports.AjvValidationError = exports.AjvSchema = exports.getAjv = exports.Ajv = exports._chunkBuffer = exports.TimeoutError = exports.HTTPError = exports.getGot = exports.inspectAnyStringifyFn = exports.inspectAny = exports.tableDiff = exports.BaseWorkerClass = exports.transformMultiThreaded = exports.writableFork = exports.writableVoid = exports.transformLimit = exports.transformLogProgress = exports.transformTap = exports.transformToArray = exports.transformToString = exports.transformSplit = exports.writablePushToArray = exports.transformNoOp = void 0;
+const tslib_1 = require("tslib");
 const ajv_1 = require("ajv");
 exports.Ajv = ajv_1.default;
 const got_1 = require("got");
@@ -118,8 +119,7 @@ const baseWorkerClass_1 = require("./stream/transform/worker/baseWorkerClass");
 Object.defineProperty(exports, "BaseWorkerClass", { enumerable: true, get: function () { return baseWorkerClass_1.BaseWorkerClass; } });
 const transformMultiThreaded_1 = require("./stream/transform/worker/transformMultiThreaded");
 Object.defineProperty(exports, "transformMultiThreaded", { enumerable: true, get: function () { return transformMultiThreaded_1.transformMultiThreaded; } });
-const writableForEach_1 = require("./stream/writable/writableForEach");
-Object.defineProperty(exports, "writableForEach", { enumerable: true, get: function () { return writableForEach_1.writableForEach; } });
+(0, tslib_1.__exportStar)(require("./stream/writable/writableForEach"), exports);
 const writableFork_1 = require("./stream/writable/writableFork");
 Object.defineProperty(exports, "writableFork", { enumerable: true, get: function () { return writableFork_1.writableFork; } });
 const writablePushToArray_1 = require("./stream/writable/writablePushToArray");

package/dist/stream/ndjson/ndJsonFileRead.js

@@ -8,7 +8,7 @@ const pipelineFromNDJsonFile_1 = require("./pipelineFromNDJsonFile");
  */
 async function ndJsonFileRead(opt) {
     const res = [];
-    await (0, pipelineFromNDJsonFile_1.pipelineFromNDJsonFile)([(0, __1.writableForEach)(r => void res.push(r))], opt);
+    await (0, pipelineFromNDJsonFile_1.pipelineFromNDJsonFile)([(0, __1.writablePushToArray)(res)], opt);
     return res;
 }
 exports.ndJsonFileRead = ndJsonFileRead;

package/dist/stream/transform/{beta/transformMap2.d.ts → legacy/transformMap.d.ts}

@@ -14,4 +14,4 @@ export declare function notNullishPredicate(item: any): boolean;
  *
  * If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
  */
-export declare function transformMap2<IN = any, OUT = IN>(mapper: AsyncMapper<IN, OUT>, opt?: TransformMapOptions<IN, OUT>): TransformTyped<IN, OUT>;
+export declare function transformMapLegacy<IN = any, OUT = IN>(mapper: AsyncMapper<IN, OUT>, opt?: TransformMapOptions<IN, OUT>): TransformTyped<IN, OUT>;

package/dist/stream/transform/legacy/transformMap.js

@@ -0,0 +1,94 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.transformMapLegacy = exports.notNullishPredicate = void 0;
+const js_lib_1 = require("@naturalcycles/js-lib");
+const through2Concurrent = require("through2-concurrent");
+const colors_1 = require("../../../colors");
+function notNullishPredicate(item) {
+    return item !== undefined && item !== null;
+}
+exports.notNullishPredicate = notNullishPredicate;
+/**
+ * Like pMap, but for streams.
+ * Inspired by `through2`.
+ * Main feature is concurrency control (implemented via `through2-concurrent`) and convenient options.
+ * Using this allows native stream .pipe() to work and use backpressure.
+ *
+ * Only works in objectMode (due to through2Concurrent).
+ *
+ * Concurrency defaults to 16.
+ *
+ * If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
+ */
+function transformMapLegacy(mapper, opt = {}) {
+    const { concurrency = 16, predicate = notNullishPredicate, errorMode = js_lib_1.ErrorMode.THROW_IMMEDIATELY, flattenArrayOutput, onError, beforeFinal, metric = 'stream', logger = console, } = opt;
+    let index = -1;
+    let isRejected = false;
+    let errors = 0;
+    const collectedErrors = []; // only used if errorMode == THROW_AGGREGATED
+    return through2Concurrent.obj({
+        maxConcurrency: concurrency,
+        // autoDestroy: true,
+        async final(cb) {
+            // console.log('transformMap final')
+            logErrorStats(logger, true);
+            await beforeFinal?.(); // call beforeFinal if defined
+            if (collectedErrors.length) {
+                // emit Aggregated error
+                cb(new js_lib_1.AggregatedError(collectedErrors));
+            }
+            else {
+                // emit no error
+                cb();
+            }
+        },
+    }, async function transformMapFn(chunk, _encoding, cb) {
+        index++;
+        // console.log({chunk, _encoding})
+        // Stop processing if THROW_IMMEDIATELY mode is used
+        if (isRejected && errorMode === js_lib_1.ErrorMode.THROW_IMMEDIATELY)
+            return cb();
+        try {
+            const currentIndex = index; // because we need to pass it to 2 functions - mapper and predicate. Refers to INPUT index (since it may return multiple outputs)
+            const res = await mapper(chunk, currentIndex);
+            const passedResults = await (0, js_lib_1.pFilter)(flattenArrayOutput && Array.isArray(res) ? res : [res], async (r) => await predicate(r, currentIndex));
+            if (passedResults.length === 0) {
+                cb(); // 0 results
+            }
+            else {
+                passedResults.forEach(r => {
+                    this.push(r);
+                    // cb(null, r)
+                });
+                cb(); // done processing
+            }
+        }
+        catch (err) {
+            logger.error(err);
+            errors++;
+            logErrorStats(logger);
+            if (onError) {
+                try {
+                    onError(err, chunk);
+                }
+                catch { }
+            }
+            if (errorMode === js_lib_1.ErrorMode.THROW_IMMEDIATELY) {
+                isRejected = true;
+                // Emit error immediately
+                return cb(err);
+            }
+            if (errorMode === js_lib_1.ErrorMode.THROW_AGGREGATED) {
+                collectedErrors.push(err);
+            }
+            // Tell input stream that we're done processing, but emit nothing to output - not error nor result
+            cb();
+        }
+    });
+    function logErrorStats(logger, final = false) {
+        if (!errors)
+            return;
+        logger.log(`${metric} ${final ? 'final ' : ''}errors: ${(0, colors_1.yellow)(errors)}`);
+    }
+}
+exports.transformMapLegacy = transformMapLegacy;

package/dist/stream/transform/transformLimit.d.ts

@@ -1,5 +1,9 @@
+import { CommonLogger } from '@naturalcycles/js-lib';
 import { TransformOptions, TransformTyped } from '../stream.model';
+export interface TransformLimitOptions extends TransformOptions {
+    logger?: CommonLogger;
+}
 /**
  * 0 or falsy value means "no limit"
  */
-export declare function transformLimit<IN>(limit?: number, opt?: TransformOptions): TransformTyped<IN, IN>;
+export declare function transformLimit<IN>(limit?: number, opt?: TransformLimitOptions): TransformTyped<IN, IN>;

package/dist/stream/transform/transformLimit.js

@@ -6,6 +6,7 @@ const stream_1 = require("stream");
  * 0 or falsy value means "no limit"
  */
 function transformLimit(limit, opt = {}) {
+    const { logger = console } = opt;
     let index = 0;
     let ended = false;
     return new stream_1.Transform({
@@ -21,7 +22,7 @@ function transformLimit(limit, opt = {}) {
             }
             if (limit && index === limit) {
                 ended = true;
-                console.log(`transformLimit: limit of ${limit} reached`);
+                logger.log(`transformLimit: limit of ${limit} reached`);
                 // this.emit('end') // this makes it "halt" on Node 14 lts
             }
         },

package/dist/stream/transform/transformLogProgress.d.ts

@@ -1,4 +1,4 @@
-import { AnyObject } from '@naturalcycles/js-lib';
+import { AnyObject, CommonLogger } from '@naturalcycles/js-lib';
 import { TransformOptions, TransformTyped } from '../stream.model';
 export interface TransformLogProgressOptions<IN = any> extends TransformOptions {
     /**
@@ -69,6 +69,7 @@ export interface TransformLogProgressOptions<IN = any> extends TransformOptions
      * @default 1000
      */
     logEvery?: number;
+    logger?: CommonLogger;
     /**
      * Function to return extra properties to the "progress object".
      *

package/dist/stream/transform/transformLogProgress.js

@@ -15,7 +15,7 @@ const inspectOpt = {
  * Pass-through transform that optionally logs progress.
  */
 function transformLogProgress(opt = {}) {
-    const { metric = 'progress', heapTotal: logHeapTotal = false, heapUsed: logHeapUsed = false, rss: logRss = true, peakRSS: logPeakRSS = true, logRPS = true, logEvery = 1000, batchSize = 1, extra, } = opt;
+    const { metric = 'progress', heapTotal: logHeapTotal = false, heapUsed: logHeapUsed = false, rss: logRss = true, peakRSS: logPeakRSS = true, logRPS = true, logEvery = 1000, batchSize = 1, extra, logger = console, } = opt;
     const logProgress = opt.logProgress !== false && logEvery !== 0; // true by default
     const logEvery10 = logEvery * 10;
     const started = Date.now();
@@ -54,7 +54,7 @@ function transformLogProgress(opt = {}) {
         const rps10 = Math.round(sma.push(lastRPS));
         if (mem.rss > peakRSS)
             peakRSS = mem.rss;
-        console.log((0, util_1.inspect)({
+        logger.log((0, util_1.inspect)({
             [final ? `${metric}_final` : metric]: batchedProgress,
             ...(extra ? extra(chunk, progress) : {}),
             ...(logHeapUsed ? { heapUsed: (0, js_lib_1._mb)(mem.heapUsed) } : {}),
@@ -76,10 +76,10 @@ function transformLogProgress(opt = {}) {
             if (perHour > 900) {
                 perHour = Math.round(perHour / 1000) + 'K';
             }
-            console.log(`${(0, colors_1.dimGrey)((0, time_lib_1.dayjs)().toPretty())} ${(0, colors_1.white)(metric)} took ${(0, colors_1.yellow)((0, js_lib_1._since)(started))} so far to process ${(0, colors_1.yellow)(batchedProgress)} rows, ~${(0, colors_1.yellow)(perHour)}/hour`);
+            logger.log(`${(0, colors_1.dimGrey)((0, time_lib_1.dayjs)().toPretty())} ${(0, colors_1.white)(metric)} took ${(0, colors_1.yellow)((0, js_lib_1._since)(started))} so far to process ${(0, colors_1.yellow)(batchedProgress)} rows, ~${(0, colors_1.yellow)(perHour)}/hour`);
         }
         else if (final) {
-            console.log(`${(0, colors_1.boldWhite)(metric)} took ${(0, colors_1.yellow)((0, js_lib_1._since)(started))} to process ${(0, colors_1.yellow)(batchedProgress)} rows with total RPS of ${(0, colors_1.yellow)(rpsTotal)}`);
+            logger.log(`${(0, colors_1.boldWhite)(metric)} took ${(0, colors_1.yellow)((0, js_lib_1._since)(started))} to process ${(0, colors_1.yellow)(batchedProgress)} rows with total RPS of ${(0, colors_1.yellow)(rpsTotal)}`);
         }
     }
 }

package/dist/stream/transform/transformMap.d.ts

@@ -1,5 +1,6 @@
 import { AsyncMapper, AsyncPredicate, CommonLogger, ErrorMode } from '@naturalcycles/js-lib';
 import { TransformTyped } from '../stream.model';
+import { transformMapLegacy } from './legacy/transformMap';
 export interface TransformMapOptions<IN = any, OUT = IN> {
     /**
      * Set true to support "multiMap" - possibility to return [] and emit 1 result for each item in the array.
@@ -44,6 +45,10 @@ export interface TransformMapOptions<IN = any, OUT = IN> {
     logger?: CommonLogger;
 }
 export declare function notNullishPredicate(item: any): boolean;
+/**
+ * Temporary export legacy transformMap, to debug 503 errors
+ */
+export declare const transformMap: typeof transformMapLegacy;
 /**
  * Like pMap, but for streams.
  * Inspired by `through2`.
@@ -56,4 +61,4 @@ export declare function notNullishPredicate(item: any): boolean;
  *
  * If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
  */
-export declare function transformMap<IN = any, OUT = IN>(mapper: AsyncMapper<IN, OUT>, opt?: TransformMapOptions<IN, OUT>): TransformTyped<IN, OUT>;
+export declare function transformMapNew<IN = any, OUT = IN>(mapper: AsyncMapper<IN, OUT>, opt?: TransformMapOptions<IN, OUT>): TransformTyped<IN, OUT>;

package/dist/stream/transform/transformMap.js

@@ -1,13 +1,18 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.transformMap = exports.notNullishPredicate = void 0;
+exports.transformMapNew = exports.transformMap = exports.notNullishPredicate = void 0;
+const stream_1 = require("stream");
 const js_lib_1 = require("@naturalcycles/js-lib");
-const through2Concurrent = require("through2-concurrent");
 const colors_1 = require("../../colors");
+const transformMap_1 = require("./legacy/transformMap");
 function notNullishPredicate(item) {
     return item !== undefined && item !== null;
 }
 exports.notNullishPredicate = notNullishPredicate;
+/**
+ * Temporary export legacy transformMap, to debug 503 errors
+ */
+exports.transformMap = transformMap_1.transformMapLegacy;
 /**
  * Like pMap, but for streams.
  * Inspired by `through2`.
@@ -20,70 +25,79 @@ exports.notNullishPredicate = notNullishPredicate;
  *
  * If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
  */
-function transformMap(mapper, opt = {}) {
+function transformMapNew(mapper, opt = {}) {
     const { concurrency = 16, predicate = notNullishPredicate, errorMode = js_lib_1.ErrorMode.THROW_IMMEDIATELY, flattenArrayOutput, onError, beforeFinal, metric = 'stream', logger = console, } = opt;
     let index = -1;
     let isRejected = false;
     let errors = 0;
     const collectedErrors = []; // only used if errorMode == THROW_AGGREGATED
-    return through2Concurrent.obj({
-        maxConcurrency: concurrency,
-        // autoDestroy: true,
+    const q = new js_lib_1.PQueue({
+        concurrency,
+        resolveOn: 'start',
+        // debug: true,
+    });
+    return new stream_1.Transform({
+        objectMode: true,
         async final(cb) {
-            // console.log('transformMap final')
+            // console.log('transformMap final', {index}, q.inFlight, q.queueSize)
+            // wait for the current inFlight jobs to complete and push their results
+            await q.onIdle();
             logErrorStats(logger, true);
             await beforeFinal?.(); // call beforeFinal if defined
             if (collectedErrors.length) {
                 // emit Aggregated error
+                // For the same reason, magically, let's not call `cb`, but emit an error event instead
+                // this.emit('error', new AggregatedError(collectedErrors))
                 cb(new js_lib_1.AggregatedError(collectedErrors));
             }
             else {
                 // emit no error
-                cb();
+                // It is truly a mistery, but calling cb() here was causing ERR_MULTIPLE_CALLBACK ?!
+                // Commenting it out seems to work ?!
+                // ?!
+                // cb()
             }
         },
-    }, async function transformMapFn(chunk, _encoding, cb) {
-        index++;
-        // console.log({chunk, _encoding})
-        // Stop processing if THROW_IMMEDIATELY mode is used
-        if (isRejected && errorMode === js_lib_1.ErrorMode.THROW_IMMEDIATELY)
-            return cb();
-        try {
-            const currentIndex = index; // because we need to pass it to 2 functions - mapper and predicate. Refers to INPUT index (since it may return multiple outputs)
-            const res = await mapper(chunk, currentIndex);
-            const passedResults = await (0, js_lib_1.pFilter)(flattenArrayOutput && Array.isArray(res) ? res : [res], async (r) => await predicate(r, currentIndex));
-            if (passedResults.length === 0) {
-                cb(); // 0 results
-            }
-            else {
-                passedResults.forEach(r => {
-                    this.push(r);
-                    // cb(null, r)
-                });
-                cb(); // done processing
-            }
-        }
-        catch (err) {
-            logger.error(err);
-            errors++;
-            logErrorStats(logger);
-            if (onError) {
+        async transform(chunk, _encoding, cb) {
+            index++;
+            // console.log('transform', {index})
+            // Stop processing if THROW_IMMEDIATELY mode is used
+            if (isRejected && errorMode === js_lib_1.ErrorMode.THROW_IMMEDIATELY)
+                return cb();
+            // It resolves when it is successfully STARTED execution.
+            // If it's queued instead - it'll wait and resolve only upon START.
+            await q.push(async () => {
                 try {
-                    onError(err, chunk);
+                    const currentIndex = index; // because we need to pass it to 2 functions - mapper and predicate. Refers to INPUT index (since it may return multiple outputs)
+                    const res = await mapper(chunk, currentIndex);
+                    const passedResults = await (0, js_lib_1.pFilter)(flattenArrayOutput && Array.isArray(res) ? res : [res], async (r) => await predicate(r, currentIndex));
+                    passedResults.forEach(r => this.push(r));
                 }
-                catch { }
-            }
-            if (errorMode === js_lib_1.ErrorMode.THROW_IMMEDIATELY) {
-                isRejected = true;
-                // Emit error immediately
-                return cb(err);
-            }
-            if (errorMode === js_lib_1.ErrorMode.THROW_AGGREGATED) {
-                collectedErrors.push(err);
-            }
-            // Tell input stream that we're done processing, but emit nothing to output - not error nor result
+                catch (err) {
+                    logger.error(err);
+                    errors++;
+                    logErrorStats(logger);
+                    if (onError) {
+                        try {
+                            onError(err, chunk);
+                        }
+                        catch { }
+                    }
+                    if (errorMode === js_lib_1.ErrorMode.THROW_IMMEDIATELY) {
+                        isRejected = true;
+                        // Emit error immediately
+                        // return cb(err as Error)
+                        return this.emit('error', err);
+                    }
+                    if (errorMode === js_lib_1.ErrorMode.THROW_AGGREGATED) {
+                        collectedErrors.push(err);
+                    }
+                }
+            });
+            // Resolved, which means it STARTED processing
+            // This means we can take more load
             cb();
-        }
+        },
     });
     function logErrorStats(logger, final = false) {
         if (!errors)
@@ -91,4 +105,4 @@ function transformMap(mapper, opt = {}) {
         logger.log(`${metric} ${final ? 'final ' : ''}errors: ${(0, colors_1.yellow)(errors)}`);
     }
 }
-exports.transformMap = transformMap;
+exports.transformMapNew = transformMapNew;

package/dist/stream/transform/transformMapSimple.d.ts

@@ -1,4 +1,4 @@
-import { ErrorMode, Mapper } from '@naturalcycles/js-lib';
+import { CommonLogger, ErrorMode, Mapper } from '@naturalcycles/js-lib';
 import { TransformTyped } from '../stream.model';
 export interface TransformMapSimpleOptions {
     /**
@@ -7,6 +7,7 @@ export interface TransformMapSimpleOptions {
      * @default ErrorMode.THROW_IMMEDIATELY
      */
     errorMode?: ErrorMode.THROW_IMMEDIATELY | ErrorMode.SUPPRESS;
+    logger?: CommonLogger;
 }
 /**
  * Simplest version of `transformMap`.

package/dist/stream/transform/transformMapSimple.js

@@ -14,7 +14,7 @@ const js_lib_1 = require("@naturalcycles/js-lib");
  */
 function transformMapSimple(mapper, opt = {}) {
     let index = -1;
-    const { errorMode = js_lib_1.ErrorMode.THROW_IMMEDIATELY } = opt;
+    const { errorMode = js_lib_1.ErrorMode.THROW_IMMEDIATELY, logger = console } = opt;
     return new stream_1.Transform({
         objectMode: true,
         transform(chunk, _encoding, cb) {
@@ -22,7 +22,7 @@ function transformMapSimple(mapper, opt = {}) {
                 cb(null, mapper(chunk, ++index));
             }
             catch (err) {
-                console.error(err);
+                logger.error(err);
                 if (errorMode === js_lib_1.ErrorMode.SUPPRESS) {
                     cb(); // suppress the error
                 }

package/dist/stream/transform/transformMapSync.d.ts

@@ -1,4 +1,4 @@
-import { ErrorMode, Mapper, Predicate } from '@naturalcycles/js-lib';
+import { CommonLogger, ErrorMode, Mapper, Predicate } from '@naturalcycles/js-lib';
 import { TransformTyped } from '../stream.model';
 export interface TransformMapSyncOptions<IN = any, OUT = IN> {
     /**
@@ -34,6 +34,7 @@ export interface TransformMapSyncOptions<IN = any, OUT = IN> {
      * @default `stream`
      */
     metric?: string;
+    logger?: CommonLogger;
 }
 /**
  * Sync (not async) version of transformMap.

package/dist/stream/transform/transformMapSync.js

@@ -11,7 +11,7 @@ const transformMap_1 = require("./transformMap");
  */
 function transformMapSync(mapper, opt = {}) {
     let index = -1;
-    const { predicate = transformMap_1.notNullishPredicate, errorMode = js_lib_1.ErrorMode.THROW_IMMEDIATELY, flattenArrayOutput = false, onError, metric = 'stream', objectMode = true, } = opt;
+    const { predicate = transformMap_1.notNullishPredicate, errorMode = js_lib_1.ErrorMode.THROW_IMMEDIATELY, flattenArrayOutput = false, onError, metric = 'stream', objectMode = true, logger = console, } = opt;
     let isRejected = false;
     let errors = 0;
     const collectedErrors = []; // only used if errorMode == THROW_AGGREGATED
@@ -39,7 +39,7 @@ function transformMapSync(mapper, opt = {}) {
                 }
             }
             catch (err) {
-                console.error(err);
+                logger.error(err);
                 errors++;
                 logErrorStats();
                 if (onError) {
@@ -75,7 +75,7 @@ function transformMapSync(mapper, opt = {}) {
     function logErrorStats(final = false) {
         if (!errors)
             return;
-        console.log(`${metric} ${final ? 'final ' : ''}errors: ${(0, colors_1.yellow)(errors)}`);
+        logger.log(`${metric} ${final ? 'final ' : ''}errors: ${(0, colors_1.yellow)(errors)}`);
     }
 }
 exports.transformMapSync = transformMapSync;

package/dist/stream/transform/transformTap.d.ts

@@ -1,9 +1,12 @@
-import { AsyncMapper } from '@naturalcycles/js-lib';
+import { AsyncMapper, CommonLogger } from '@naturalcycles/js-lib';
 import { TransformOptions, TransformTyped } from '../stream.model';
+export interface TransformTapOptions extends TransformOptions {
+    logger?: CommonLogger;
+}
 /**
  * Similar to RxJS `tap` - allows to run a function for each stream item, without affecting the result.
  * Item is passed through to the output.
  *
  * Can also act as a counter, since `index` is passed to `fn`
  */
-export declare function transformTap<IN>(fn: AsyncMapper<IN, any>, opt?: TransformOptions): TransformTyped<IN, IN>;
+export declare function transformTap<IN>(fn: AsyncMapper<IN, any>, opt?: TransformTapOptions): TransformTyped<IN, IN>;

package/dist/stream/transform/transformTap.js

@@ -9,6 +9,7 @@ const stream_1 = require("stream");
  * Can also act as a counter, since `index` is passed to `fn`
  */
 function transformTap(fn, opt = {}) {
+    const { logger = console } = opt;
     let index = 0;
     return new stream_1.Transform({
         objectMode: true,
@@ -19,7 +20,7 @@ function transformTap(fn, opt = {}) {
                 await fn(chunk, index++);
             }
             catch (err) {
-                console.error(err);
+                logger.error(err);
                 // suppressed error
             }
             cb(null, chunk); // pass through the item

package/dist/stream/writable/writableForEach.d.ts

@@ -1,7 +1,11 @@
-import { AsyncMapper } from '@naturalcycles/js-lib';
+import { AsyncMapper, Mapper } from '@naturalcycles/js-lib';
 import { TransformMapOptions } from '../..';
 import { WritableTyped } from '../stream.model';
 /**
  * Just an alias to transformMap that declares OUT as void.
  */
 export declare function writableForEach<IN = any>(mapper: AsyncMapper<IN, void>, opt?: TransformMapOptions<IN, void>): WritableTyped<IN>;
+/**
+ * Just an alias to transformMap that declares OUT as void.
+ */
+export declare function writableForEachSync<IN = any>(mapper: Mapper<IN, void>, opt?: TransformMapOptions<IN, void>): WritableTyped<IN>;

package/dist/stream/writable/writableForEach.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.writableForEach = void 0;
+exports.writableForEachSync = exports.writableForEach = void 0;
 const js_lib_1 = require("@naturalcycles/js-lib");
 const __1 = require("../..");
 /**
@@ -10,3 +10,10 @@ function writableForEach(mapper, opt = {}) {
     return (0, __1.transformMap)(mapper, { ...opt, predicate: js_lib_1._passNothingPredicate });
 }
 exports.writableForEach = writableForEach;
+/**
+ * Just an alias to transformMap that declares OUT as void.
+ */
+function writableForEachSync(mapper, opt = {}) {
+    return (0, __1.transformMapSync)(mapper, { ...opt, predicate: js_lib_1._passNothingPredicate });
+}
+exports.writableForEachSync = writableForEachSync;

package/dist/stream/writable/writableFork.d.ts

@@ -5,5 +5,7 @@ import { TransformOptions, WritableTyped } from '../stream.model';
  * Currently does NOT (!) maintain backpressure.
  * Error in the forked pipeline will propagate up to the main pipeline (and log error, to be sure).
  * Will wait until all forked pipelines are completed before completing the stream.
+ *
+ * @experimental
  */
 export declare function writableFork<T>(chains: NodeJS.WritableStream[][], opt?: TransformOptions): WritableTyped<T>;

package/dist/stream/writable/writableFork.js

@@ -8,6 +8,8 @@ const __1 = require("../..");
  * Currently does NOT (!) maintain backpressure.
  * Error in the forked pipeline will propagate up to the main pipeline (and log error, to be sure).
  * Will wait until all forked pipelines are completed before completing the stream.
+ *
+ * @experimental
  */
 function writableFork(chains, opt) {
     const readables = [];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@naturalcycles/nodejs-lib",
-  "version": "12.55.1",
+  "version": "12.58.0",
   "scripts": {
     "prepare": "husky install",
     "docs-serve": "vuepress dev docs",

package/src/index.ts

@@ -104,7 +104,7 @@ import {
   TransformMultiThreadedOptions,
 } from './stream/transform/worker/transformMultiThreaded'
 import { WorkerInput, WorkerOutput } from './stream/transform/worker/transformMultiThreaded.model'
-import { writableForEach } from './stream/writable/writableForEach'
+export * from './stream/writable/writableForEach'
 import { writableFork } from './stream/writable/writableFork'
 import { writablePushToArray } from './stream/writable/writablePushToArray'
 import { writableVoid } from './stream/writable/writableVoid'
@@ -327,7 +327,6 @@ export {
   transformMapSync,
   transformMapSimple,
   transformNoOp,
-  writableForEach,
   writablePushToArray,
   transformSplit,
   transformToString,

package/src/stream/ndjson/ndJsonFileRead.ts

@@ -1,4 +1,4 @@
-import { writableForEach } from '../..'
+import { writablePushToArray } from '../..'
 import { pipelineFromNDJsonFile, PipelineFromNDJsonFileOptions } from './pipelineFromNDJsonFile'
 
 /**
@@ -9,7 +9,7 @@ export async function ndJsonFileRead<OUT = any>(
 ): Promise<OUT[]> {
   const res: OUT[] = []
 
-  await pipelineFromNDJsonFile([writableForEach(r => void res.push(r))], opt)
+  await pipelineFromNDJsonFile([writablePushToArray(res)], opt)
 
   return res
 }