@naturalcycles/nodejs-lib 12.55.1 → 12.56.0

package/dist/index.d.ts

@@ -47,7 +47,7 @@ import { transformToString } from './stream/transform/transformToString';
 import { BaseWorkerClass, WorkerClassInterface } from './stream/transform/worker/baseWorkerClass';
 import { transformMultiThreaded, TransformMultiThreadedOptions } from './stream/transform/worker/transformMultiThreaded';
 import { WorkerInput, WorkerOutput } from './stream/transform/worker/transformMultiThreaded.model';
-import { writableForEach } from './stream/writable/writableForEach';
+export * from './stream/writable/writableForEach';
 import { writableFork } from './stream/writable/writableFork';
 import { writablePushToArray } from './stream/writable/writablePushToArray';
 import { writableVoid } from './stream/writable/writableVoid';
@@ -67,4 +67,4 @@ import { convert, getValidationResult, isValid, JoiValidationResult, undefinedIf
 import { sanitizeHTML, SanitizeHTMLOptions } from './validation/sanitize.util';
 import { runScript, RunScriptOptions } from './script';
 export type { RunScriptOptions, JoiValidationErrorData, JoiValidationResult, ValidationErrorItem, ExtendedJoi, SchemaTyped, AnySchema, AnySchemaTyped, ArraySchemaTyped, BooleanSchemaTyped, NumberSchemaTyped, ObjectSchemaTyped, StringSchemaTyped, IDebug, IDebugger, SlackServiceCfg, SlackMessage, SlackMessageProps, SlackApiBody, SlackMessagePrefixHook, ReadableTyped, WritableTyped, TransformTyped, PipelineFromNDJsonFileOptions, PipelineToNDJsonFileOptions, TransformJsonParseOptions, TransformToNDJsonOptions, TransformMapOptions, TransformMapSyncOptions, NDJSONStreamForEachOptions, TransformOptions, TransformLogProgressOptions, TransformMultiThreadedOptions, WorkerClassInterface, WorkerInput, WorkerOutput, TableDiffOptions, InspectAnyOptions, Got, GetGotOptions, AfterResponseHook, BeforeErrorHook, BeforeRequestHook, AjvValidationOptions, AjvSchemaCfg, AjvValidationErrorData, SanitizeHTMLOptions, };
-export { JoiValidationError, validate, getValidationResult, isValid, undefinedIfInvalid, convert, Joi, booleanSchema, booleanDefaultToFalseSchema, stringSchema, numberSchema, integerSchema, percentageSchema, dateStringSchema, arraySchema, binarySchema, objectSchema, oneOfSchema, anySchema, anyObjectSchema, baseDBEntitySchema, savedDBEntitySchema, idSchema, unixTimestampSchema, verSchema, emailSchema, SEM_VER_PATTERN, semVerSchema, userAgentSchema, utcOffsetSchema, ipAddressSchema, slugSchema, urlSchema, processSharedUtil, zipBuffer, gzipBuffer, unzipBuffer, gunzipBuffer, zipString, gzipString, unzipToString, gunzipToString, requireEnvKeys, requireFileToExist, LRUMemoCache, stringId, stringIdAsync, stringIdUnsafe, ALPHABET_NUMBER, ALPHABET_LOWERCASE, ALPHABET_UPPERCASE, ALPHABET_ALPHANUMERIC_LOWERCASE, ALPHABET_ALPHANUMERIC_UPPERCASE, ALPHABET_ALPHANUMERIC, md5, hash, hashAsBuffer, md5AsBuffer, stringToBase64, base64ToString, bufferToBase64, base64ToBuffer, Debug, getSecretMap, setSecretMap, loadSecretsFromEnv, loadSecretsFromJsonFile, removeSecretsFromEnv, secret, secretOptional, memoryUsage, memoryUsageFull, SlackService, slackDefaultMessagePrefixHook, readableCreate, readableFrom, readableFromArray, readableToArray, readableForEach, readableForEachSync, readableMap, readableMapToArray, _pipeline, transformBuffer, ndjsonMap, ndJsonFileRead, ndJsonFileWrite, ndjsonStreamForEach, pipelineFromNDJsonFile, pipelineToNDJsonFile, NDJsonStats, streamToNDJsonFile, transformJsonParse, bufferReviver, transformToNDJson, transformFilter, transformFilterSync, transformMap, transformMapSync, transformMapSimple, transformNoOp, writableForEach, writablePushToArray, transformSplit, transformToString, transformToArray, transformTap, transformLogProgress, transformLimit, writableVoid, writableFork, transformMultiThreaded, BaseWorkerClass, tableDiff, inspectAny, inspectAnyStringifyFn, getGot, HTTPError, TimeoutError, _chunkBuffer, Ajv, getAjv, AjvSchema, AjvValidationError, readJsonSchemas, readAjvSchemas, hasColors, sanitizeHTML, runScript, };
+export { JoiValidationError, validate, getValidationResult, isValid, undefinedIfInvalid, convert, Joi, booleanSchema, booleanDefaultToFalseSchema, stringSchema, numberSchema, integerSchema, percentageSchema, dateStringSchema, arraySchema, binarySchema, objectSchema, oneOfSchema, anySchema, anyObjectSchema, baseDBEntitySchema, savedDBEntitySchema, idSchema, unixTimestampSchema, verSchema, emailSchema, SEM_VER_PATTERN, semVerSchema, userAgentSchema, utcOffsetSchema, ipAddressSchema, slugSchema, urlSchema, processSharedUtil, zipBuffer, gzipBuffer, unzipBuffer, gunzipBuffer, zipString, gzipString, unzipToString, gunzipToString, requireEnvKeys, requireFileToExist, LRUMemoCache, stringId, stringIdAsync, stringIdUnsafe, ALPHABET_NUMBER, ALPHABET_LOWERCASE, ALPHABET_UPPERCASE, ALPHABET_ALPHANUMERIC_LOWERCASE, ALPHABET_ALPHANUMERIC_UPPERCASE, ALPHABET_ALPHANUMERIC, md5, hash, hashAsBuffer, md5AsBuffer, stringToBase64, base64ToString, bufferToBase64, base64ToBuffer, Debug, getSecretMap, setSecretMap, loadSecretsFromEnv, loadSecretsFromJsonFile, removeSecretsFromEnv, secret, secretOptional, memoryUsage, memoryUsageFull, SlackService, slackDefaultMessagePrefixHook, readableCreate, readableFrom, readableFromArray, readableToArray, readableForEach, readableForEachSync, readableMap, readableMapToArray, _pipeline, transformBuffer, ndjsonMap, ndJsonFileRead, ndJsonFileWrite, ndjsonStreamForEach, pipelineFromNDJsonFile, pipelineToNDJsonFile, NDJsonStats, streamToNDJsonFile, transformJsonParse, bufferReviver, transformToNDJson, transformFilter, transformFilterSync, transformMap, transformMapSync, transformMapSimple, transformNoOp, writablePushToArray, transformSplit, transformToString, transformToArray, transformTap, transformLogProgress, transformLimit, writableVoid, writableFork, transformMultiThreaded, BaseWorkerClass, tableDiff, inspectAny, inspectAnyStringifyFn, getGot, HTTPError, TimeoutError, _chunkBuffer, Ajv, getAjv, AjvSchema, AjvValidationError, readJsonSchemas, readAjvSchemas, hasColors, sanitizeHTML, runScript, };

package/dist/index.js

@@ -2,7 +2,8 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ALPHABET_LOWERCASE = exports.ALPHABET_NUMBER = exports.stringIdUnsafe = exports.stringIdAsync = exports.stringId = exports.LRUMemoCache = exports.requireFileToExist = exports.requireEnvKeys = exports.gunzipToString = exports.unzipToString = exports.gzipString = exports.zipString = exports.gunzipBuffer = exports.unzipBuffer = exports.gzipBuffer = exports.zipBuffer = exports.processSharedUtil = exports.urlSchema = exports.slugSchema = exports.ipAddressSchema = exports.utcOffsetSchema = exports.userAgentSchema = exports.semVerSchema = exports.SEM_VER_PATTERN = exports.emailSchema = exports.verSchema = exports.unixTimestampSchema = exports.idSchema = exports.savedDBEntitySchema = exports.baseDBEntitySchema = exports.anyObjectSchema = exports.anySchema = exports.oneOfSchema = exports.objectSchema = exports.binarySchema = exports.arraySchema = exports.dateStringSchema = exports.percentageSchema = exports.integerSchema = exports.numberSchema = exports.stringSchema = exports.booleanDefaultToFalseSchema = exports.booleanSchema = exports.Joi = exports.convert = exports.undefinedIfInvalid = exports.isValid = exports.getValidationResult = exports.validate = exports.JoiValidationError = void 0;
 exports.transformMapSimple = exports.transformMapSync = exports.transformMap = exports.transformFilterSync = exports.transformFilter = exports.transformToNDJson = exports.bufferReviver = exports.transformJsonParse = exports.streamToNDJsonFile = exports.NDJsonStats = exports.pipelineToNDJsonFile = exports.pipelineFromNDJsonFile = exports.ndjsonStreamForEach = exports.ndJsonFileWrite = exports.ndJsonFileRead = exports.ndjsonMap = exports.transformBuffer = exports._pipeline = exports.readableMapToArray = exports.readableMap = exports.readableForEachSync = exports.readableForEach = exports.readableToArray = exports.readableFromArray = exports.readableFrom = exports.readableCreate = exports.slackDefaultMessagePrefixHook = exports.SlackService = exports.memoryUsageFull = exports.memoryUsage = exports.secretOptional = exports.secret = exports.removeSecretsFromEnv = exports.loadSecretsFromJsonFile = exports.loadSecretsFromEnv = exports.setSecretMap = exports.getSecretMap = exports.Debug = exports.base64ToBuffer = exports.bufferToBase64 = exports.base64ToString = exports.stringToBase64 = exports.md5AsBuffer = exports.hashAsBuffer = exports.hash = exports.md5 = exports.ALPHABET_ALPHANUMERIC = exports.ALPHABET_ALPHANUMERIC_UPPERCASE = exports.ALPHABET_ALPHANUMERIC_LOWERCASE = exports.ALPHABET_UPPERCASE = void 0;
-exports.runScript = exports.sanitizeHTML = exports.hasColors = exports.readAjvSchemas = exports.readJsonSchemas = exports.AjvValidationError = exports.AjvSchema = exports.getAjv = exports.Ajv = exports._chunkBuffer = exports.TimeoutError = exports.HTTPError = exports.getGot = exports.inspectAnyStringifyFn = exports.inspectAny = exports.tableDiff = exports.BaseWorkerClass = exports.transformMultiThreaded = exports.writableFork = exports.writableVoid = exports.transformLimit = exports.transformLogProgress = exports.transformTap = exports.transformToArray = exports.transformToString = exports.transformSplit = exports.writablePushToArray = exports.writableForEach = exports.transformNoOp = void 0;
+exports.runScript = exports.sanitizeHTML = exports.hasColors = exports.readAjvSchemas = exports.readJsonSchemas = exports.AjvValidationError = exports.AjvSchema = exports.getAjv = exports.Ajv = exports._chunkBuffer = exports.TimeoutError = exports.HTTPError = exports.getGot = exports.inspectAnyStringifyFn = exports.inspectAny = exports.tableDiff = exports.BaseWorkerClass = exports.transformMultiThreaded = exports.writableFork = exports.writableVoid = exports.transformLimit = exports.transformLogProgress = exports.transformTap = exports.transformToArray = exports.transformToString = exports.transformSplit = exports.writablePushToArray = exports.transformNoOp = void 0;
+const tslib_1 = require("tslib");
 const ajv_1 = require("ajv");
 exports.Ajv = ajv_1.default;
 const got_1 = require("got");
@@ -118,8 +119,7 @@ const baseWorkerClass_1 = require("./stream/transform/worker/baseWorkerClass");
 Object.defineProperty(exports, "BaseWorkerClass", { enumerable: true, get: function () { return baseWorkerClass_1.BaseWorkerClass; } });
 const transformMultiThreaded_1 = require("./stream/transform/worker/transformMultiThreaded");
 Object.defineProperty(exports, "transformMultiThreaded", { enumerable: true, get: function () { return transformMultiThreaded_1.transformMultiThreaded; } });
-const writableForEach_1 = require("./stream/writable/writableForEach");
-Object.defineProperty(exports, "writableForEach", { enumerable: true, get: function () { return writableForEach_1.writableForEach; } });
+(0, tslib_1.__exportStar)(require("./stream/writable/writableForEach"), exports);
 const writableFork_1 = require("./stream/writable/writableFork");
 Object.defineProperty(exports, "writableFork", { enumerable: true, get: function () { return writableFork_1.writableFork; } });
 const writablePushToArray_1 = require("./stream/writable/writablePushToArray");

package/dist/stream/ndjson/ndJsonFileRead.js

@@ -8,7 +8,7 @@ const pipelineFromNDJsonFile_1 = require("./pipelineFromNDJsonFile");
  */
 async function ndJsonFileRead(opt) {
     const res = [];
-    await (0, pipelineFromNDJsonFile_1.pipelineFromNDJsonFile)([(0, __1.writableForEach)(r => void res.push(r))], opt);
+    await (0, pipelineFromNDJsonFile_1.pipelineFromNDJsonFile)([(0, __1.writablePushToArray)(res)], opt);
     return res;
 }
 exports.ndJsonFileRead = ndJsonFileRead;

package/dist/stream/transform/{beta/transformMap2.d.ts → legacy/transformMap.d.ts}

@@ -14,4 +14,4 @@ export declare function notNullishPredicate(item: any): boolean;
  *
  * If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
  */
-export declare function transformMap2<IN = any, OUT = IN>(mapper: AsyncMapper<IN, OUT>, opt?: TransformMapOptions<IN, OUT>): TransformTyped<IN, OUT>;
+export declare function transformMapLegacy<IN = any, OUT = IN>(mapper: AsyncMapper<IN, OUT>, opt?: TransformMapOptions<IN, OUT>): TransformTyped<IN, OUT>;

package/dist/stream/transform/legacy/transformMap.js

@@ -0,0 +1,94 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.transformMapLegacy = exports.notNullishPredicate = void 0;
+const js_lib_1 = require("@naturalcycles/js-lib");
+const through2Concurrent = require("through2-concurrent");
+const colors_1 = require("../../../colors");
+function notNullishPredicate(item) {
+    return item !== undefined && item !== null;
+}
+exports.notNullishPredicate = notNullishPredicate;
+/**
+ * Like pMap, but for streams.
+ * Inspired by `through2`.
+ * Main feature is concurrency control (implemented via `through2-concurrent`) and convenient options.
+ * Using this allows native stream .pipe() to work and use backpressure.
+ *
+ * Only works in objectMode (due to through2Concurrent).
+ *
+ * Concurrency defaults to 16.
+ *
+ * If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
+ */
+function transformMapLegacy(mapper, opt = {}) {
+    const { concurrency = 16, predicate = notNullishPredicate, errorMode = js_lib_1.ErrorMode.THROW_IMMEDIATELY, flattenArrayOutput, onError, beforeFinal, metric = 'stream', logger = console, } = opt;
+    let index = -1;
+    let isRejected = false;
+    let errors = 0;
+    const collectedErrors = []; // only used if errorMode == THROW_AGGREGATED
+    return through2Concurrent.obj({
+        maxConcurrency: concurrency,
+        // autoDestroy: true,
+        async final(cb) {
+            // console.log('transformMap final')
+            logErrorStats(logger, true);
+            await beforeFinal?.(); // call beforeFinal if defined
+            if (collectedErrors.length) {
+                // emit Aggregated error
+                cb(new js_lib_1.AggregatedError(collectedErrors));
+            }
+            else {
+                // emit no error
+                cb();
+            }
+        },
+    }, async function transformMapFn(chunk, _encoding, cb) {
+        index++;
+        // console.log({chunk, _encoding})
+        // Stop processing if THROW_IMMEDIATELY mode is used
+        if (isRejected && errorMode === js_lib_1.ErrorMode.THROW_IMMEDIATELY)
+            return cb();
+        try {
+            const currentIndex = index; // because we need to pass it to 2 functions - mapper and predicate. Refers to INPUT index (since it may return multiple outputs)
+            const res = await mapper(chunk, currentIndex);
+            const passedResults = await (0, js_lib_1.pFilter)(flattenArrayOutput && Array.isArray(res) ? res : [res], async (r) => await predicate(r, currentIndex));
+            if (passedResults.length === 0) {
+                cb(); // 0 results
+            }
+            else {
+                passedResults.forEach(r => {
+                    this.push(r);
+                    // cb(null, r)
+                });
+                cb(); // done processing
+            }
+        }
+        catch (err) {
+            logger.error(err);
+            errors++;
+            logErrorStats(logger);
+            if (onError) {
+                try {
+                    onError(err, chunk);
+                }
+                catch { }
+            }
+            if (errorMode === js_lib_1.ErrorMode.THROW_IMMEDIATELY) {
+                isRejected = true;
+                // Emit error immediately
+                return cb(err);
+            }
+            if (errorMode === js_lib_1.ErrorMode.THROW_AGGREGATED) {
+                collectedErrors.push(err);
+            }
+            // Tell input stream that we're done processing, but emit nothing to output - not error nor result
+            cb();
+        }
+    });
+    function logErrorStats(logger, final = false) {
+        if (!errors)
+            return;
+        logger.log(`${metric} ${final ? 'final ' : ''}errors: ${(0, colors_1.yellow)(errors)}`);
+    }
+}
+exports.transformMapLegacy = transformMapLegacy;

package/dist/stream/transform/transformMap.js

@@ -1,8 +1,8 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.transformMap = exports.notNullishPredicate = void 0;
+const stream_1 = require("stream");
 const js_lib_1 = require("@naturalcycles/js-lib");
-const through2Concurrent = require("through2-concurrent");
 const colors_1 = require("../../colors");
 function notNullishPredicate(item) {
     return item !== undefined && item !== null;
@@ -26,11 +26,17 @@ function transformMap(mapper, opt = {}) {
     let isRejected = false;
     let errors = 0;
     const collectedErrors = []; // only used if errorMode == THROW_AGGREGATED
-    return through2Concurrent.obj({
-        maxConcurrency: concurrency,
-        // autoDestroy: true,
+    const q = new js_lib_1.PQueue({
+        concurrency,
+        resolveOn: 'start',
+        // debug: true,
+    });
+    return new stream_1.Transform({
+        objectMode: true,
         async final(cb) {
-            // console.log('transformMap final')
+            // console.log('transformMap final', {index}, q.inFlight, q.queueSize)
+            // wait for the current inFlight jobs to complete and push their results
+            await q.onIdle();
             logErrorStats(logger, true);
             await beforeFinal?.(); // call beforeFinal if defined
             if (collectedErrors.length) {
@@ -42,48 +48,46 @@ function transformMap(mapper, opt = {}) {
                 cb();
             }
         },
-    }, async function transformMapFn(chunk, _encoding, cb) {
-        index++;
-        // console.log({chunk, _encoding})
-        // Stop processing if THROW_IMMEDIATELY mode is used
-        if (isRejected && errorMode === js_lib_1.ErrorMode.THROW_IMMEDIATELY)
-            return cb();
-        try {
-            const currentIndex = index; // because we need to pass it to 2 functions - mapper and predicate. Refers to INPUT index (since it may return multiple outputs)
-            const res = await mapper(chunk, currentIndex);
-            const passedResults = await (0, js_lib_1.pFilter)(flattenArrayOutput && Array.isArray(res) ? res : [res], async (r) => await predicate(r, currentIndex));
-            if (passedResults.length === 0) {
-                cb(); // 0 results
-            }
-            else {
-                passedResults.forEach(r => {
-                    this.push(r);
-                    // cb(null, r)
-                });
-                cb(); // done processing
-            }
-        }
-        catch (err) {
-            logger.error(err);
-            errors++;
-            logErrorStats(logger);
-            if (onError) {
+        async transform(chunk, _encoding, cb) {
+            index++;
+            // console.log('transform', {index})
+            // Stop processing if THROW_IMMEDIATELY mode is used
+            if (isRejected && errorMode === js_lib_1.ErrorMode.THROW_IMMEDIATELY)
+                return cb();
+            // It resolves when it is successfully STARTED execution.
+            // If it's queued instead - it'll wait and resolve only upon START.
+            await q.push(async () => {
                 try {
-                    onError(err, chunk);
+                    const currentIndex = index; // because we need to pass it to 2 functions - mapper and predicate. Refers to INPUT index (since it may return multiple outputs)
+                    const res = await mapper(chunk, currentIndex);
+                    const passedResults = await (0, js_lib_1.pFilter)(flattenArrayOutput && Array.isArray(res) ? res : [res], async (r) => await predicate(r, currentIndex));
+                    passedResults.forEach(r => this.push(r));
                 }
-                catch { }
-            }
-            if (errorMode === js_lib_1.ErrorMode.THROW_IMMEDIATELY) {
-                isRejected = true;
-                // Emit error immediately
-                return cb(err);
-            }
-            if (errorMode === js_lib_1.ErrorMode.THROW_AGGREGATED) {
-                collectedErrors.push(err);
-            }
-            // Tell input stream that we're done processing, but emit nothing to output - not error nor result
+                catch (err) {
+                    logger.error(err);
+                    errors++;
+                    logErrorStats(logger);
+                    if (onError) {
+                        try {
+                            onError(err, chunk);
+                        }
+                        catch { }
+                    }
+                    if (errorMode === js_lib_1.ErrorMode.THROW_IMMEDIATELY) {
+                        isRejected = true;
+                        // Emit error immediately
+                        // return cb(err as Error)
+                        return this.emit('error', err);
+                    }
+                    if (errorMode === js_lib_1.ErrorMode.THROW_AGGREGATED) {
+                        collectedErrors.push(err);
+                    }
+                }
+            });
+            // Resolved, which means it STARTED processing
+            // This means we can take more load
             cb();
-        }
+        },
     });
     function logErrorStats(logger, final = false) {
         if (!errors)

package/dist/stream/writable/writableForEach.d.ts

@@ -1,7 +1,11 @@
-import { AsyncMapper } from '@naturalcycles/js-lib';
+import { AsyncMapper, Mapper } from '@naturalcycles/js-lib';
 import { TransformMapOptions } from '../..';
 import { WritableTyped } from '../stream.model';
 /**
  * Just an alias to transformMap that declares OUT as void.
  */
 export declare function writableForEach<IN = any>(mapper: AsyncMapper<IN, void>, opt?: TransformMapOptions<IN, void>): WritableTyped<IN>;
+/**
+ * Just an alias to transformMap that declares OUT as void.
+ */
+export declare function writableForEachSync<IN = any>(mapper: Mapper<IN, void>, opt?: TransformMapOptions<IN, void>): WritableTyped<IN>;

package/dist/stream/writable/writableForEach.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.writableForEach = void 0;
+exports.writableForEachSync = exports.writableForEach = void 0;
 const js_lib_1 = require("@naturalcycles/js-lib");
 const __1 = require("../..");
 /**
@@ -10,3 +10,10 @@ function writableForEach(mapper, opt = {}) {
     return (0, __1.transformMap)(mapper, { ...opt, predicate: js_lib_1._passNothingPredicate });
 }
 exports.writableForEach = writableForEach;
+/**
+ * Just an alias to transformMap that declares OUT as void.
+ */
+function writableForEachSync(mapper, opt = {}) {
+    return (0, __1.transformMapSync)(mapper, { ...opt, predicate: js_lib_1._passNothingPredicate });
+}
+exports.writableForEachSync = writableForEachSync;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@naturalcycles/nodejs-lib",
-  "version": "12.55.1",
+  "version": "12.56.0",
   "scripts": {
     "prepare": "husky install",
     "docs-serve": "vuepress dev docs",

package/src/index.ts

@@ -104,7 +104,7 @@ import {
   TransformMultiThreadedOptions,
 } from './stream/transform/worker/transformMultiThreaded'
 import { WorkerInput, WorkerOutput } from './stream/transform/worker/transformMultiThreaded.model'
-import { writableForEach } from './stream/writable/writableForEach'
+export * from './stream/writable/writableForEach'
 import { writableFork } from './stream/writable/writableFork'
 import { writablePushToArray } from './stream/writable/writablePushToArray'
 import { writableVoid } from './stream/writable/writableVoid'
@@ -327,7 +327,6 @@ export {
   transformMapSync,
   transformMapSimple,
   transformNoOp,
-  writableForEach,
   writablePushToArray,
   transformSplit,
   transformToString,

package/src/stream/ndjson/ndJsonFileRead.ts

@@ -1,4 +1,4 @@
-import { writableForEach } from '../..'
+import { writablePushToArray } from '../..'
 import { pipelineFromNDJsonFile, PipelineFromNDJsonFileOptions } from './pipelineFromNDJsonFile'
 
 /**
@@ -9,7 +9,7 @@ export async function ndJsonFileRead<OUT = any>(
 ): Promise<OUT[]> {
   const res: OUT[] = []
 
-  await pipelineFromNDJsonFile([writableForEach(r => void res.push(r))], opt)
+  await pipelineFromNDJsonFile([writablePushToArray(res)], opt)
 
   return res
 }

package/src/stream/transform/legacy/transformMap.ts

@@ -0,0 +1,133 @@
+import { Transform } from 'stream'
+import {
+  AggregatedError,
+  AsyncMapper,
+  CommonLogger,
+  ErrorMode,
+  pFilter,
+} from '@naturalcycles/js-lib'
+import through2Concurrent = require('through2-concurrent')
+import { yellow } from '../../../colors'
+import { TransformTyped } from '../../stream.model'
+import { TransformMapOptions } from '../transformMap'
+
+export function notNullishPredicate(item: any): boolean {
+  return item !== undefined && item !== null
+}
+
+/**
+ * Like pMap, but for streams.
+ * Inspired by `through2`.
+ * Main feature is concurrency control (implemented via `through2-concurrent`) and convenient options.
+ * Using this allows native stream .pipe() to work and use backpressure.
+ *
+ * Only works in objectMode (due to through2Concurrent).
+ *
+ * Concurrency defaults to 16.
+ *
+ * If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
+ */
+export function transformMapLegacy<IN = any, OUT = IN>(
+  mapper: AsyncMapper<IN, OUT>,
+  opt: TransformMapOptions<IN, OUT> = {},
+): TransformTyped<IN, OUT> {
+  const {
+    concurrency = 16,
+    predicate = notNullishPredicate,
+    errorMode = ErrorMode.THROW_IMMEDIATELY,
+    flattenArrayOutput,
+    onError,
+    beforeFinal,
+    metric = 'stream',
+    logger = console,
+  } = opt
+
+  let index = -1
+  let isRejected = false
+  let errors = 0
+  const collectedErrors: Error[] = [] // only used if errorMode == THROW_AGGREGATED
+
+  return through2Concurrent.obj(
+    {
+      maxConcurrency: concurrency,
+      // autoDestroy: true,
+      async final(cb) {
+        // console.log('transformMap final')
+
+        logErrorStats(logger, true)
+
+        await beforeFinal?.() // call beforeFinal if defined
+
+        if (collectedErrors.length) {
+          // emit Aggregated error
+          cb(new AggregatedError(collectedErrors))
+        } else {
+          // emit no error
+          cb()
+        }
+      },
+    },
+    async function transformMapFn(
+      this: Transform,
+      chunk: IN,
+      _encoding: any,
+      cb: (...args: any[]) => any,
+    ) {
+      index++
+      // console.log({chunk, _encoding})
+
+      // Stop processing if THROW_IMMEDIATELY mode is used
+      if (isRejected && errorMode === ErrorMode.THROW_IMMEDIATELY) return cb()
+
+      try {
+        const currentIndex = index // because we need to pass it to 2 functions - mapper and predicate. Refers to INPUT index (since it may return multiple outputs)
+        const res = await mapper(chunk, currentIndex)
+        const passedResults = await pFilter(
+          flattenArrayOutput && Array.isArray(res) ? res : [res],
+          async r => await predicate(r, currentIndex),
+        )
+
+        if (passedResults.length === 0) {
+          cb() // 0 results
+        } else {
+          passedResults.forEach(r => {
+            this.push(r)
+            // cb(null, r)
+          })
+          cb() // done processing
+        }
+      } catch (err) {
+        logger.error(err)
+
+        errors++
+
+        logErrorStats(logger)
+
+        if (onError) {
+          try {
+            onError(err, chunk)
+          } catch {}
+        }
+
+        if (errorMode === ErrorMode.THROW_IMMEDIATELY) {
+          isRejected = true
+          // Emit error immediately
+          return cb(err)
+        }
+
+        if (errorMode === ErrorMode.THROW_AGGREGATED) {
+          collectedErrors.push(err as Error)
+        }
+
+        // Tell input stream that we're done processing, but emit nothing to output - not error nor result
+        cb()
+      }
+    },
+  )
+
+  function logErrorStats(logger: CommonLogger, final = false): void {
+    if (!errors) return
+
+    logger.log(`${metric} ${final ? 'final ' : ''}errors: ${yellow(errors)}`)
+  }
+}

package/src/stream/transform/transformMap.ts

@@ -6,8 +6,8 @@ import {
   CommonLogger,
   ErrorMode,
   pFilter,
+  PQueue,
 } from '@naturalcycles/js-lib'
-import through2Concurrent = require('through2-concurrent')
 import { yellow } from '../../colors'
 import { TransformTyped } from '../stream.model'
 
@@ -98,83 +98,84 @@ export function transformMap<IN = any, OUT = IN>(
   let errors = 0
   const collectedErrors: Error[] = [] // only used if errorMode == THROW_AGGREGATED
 
-  return through2Concurrent.obj(
-    {
-      maxConcurrency: concurrency,
-      // autoDestroy: true,
-      async final(cb) {
-        // console.log('transformMap final')
+  const q = new PQueue({
+    concurrency,
+    resolveOn: 'start',
+    // debug: true,
+  })
 
-        logErrorStats(logger, true)
+  return new Transform({
+    objectMode: true,
 
-        await beforeFinal?.() // call beforeFinal if defined
+    async final(cb) {
+      // console.log('transformMap final', {index}, q.inFlight, q.queueSize)
 
-        if (collectedErrors.length) {
-          // emit Aggregated error
-          cb(new AggregatedError(collectedErrors))
-        } else {
-          // emit no error
-          cb()
-        }
-      },
-    },
-    async function transformMapFn(
-      this: Transform,
-      chunk: IN,
-      _encoding: any,
-      cb: (...args: any[]) => any,
-    ) {
-      index++
-      // console.log({chunk, _encoding})
+      // wait for the current inFlight jobs to complete and push their results
+      await q.onIdle()
 
-      // Stop processing if THROW_IMMEDIATELY mode is used
-      if (isRejected && errorMode === ErrorMode.THROW_IMMEDIATELY) return cb()
+      logErrorStats(logger, true)
 
-      try {
-        const currentIndex = index // because we need to pass it to 2 functions - mapper and predicate. Refers to INPUT index (since it may return multiple outputs)
-        const res = await mapper(chunk, currentIndex)
-        const passedResults = await pFilter(
-          flattenArrayOutput && Array.isArray(res) ? res : [res],
-          async r => await predicate(r, currentIndex),
-        )
-
-        if (passedResults.length === 0) {
-          cb() // 0 results
-        } else {
-          passedResults.forEach(r => {
-            this.push(r)
-            // cb(null, r)
-          })
-          cb() // done processing
-        }
-      } catch (err) {
-        logger.error(err)
-
-        errors++
+      await beforeFinal?.() // call beforeFinal if defined
 
-        logErrorStats(logger)
+      if (collectedErrors.length) {
+        // emit Aggregated error
+        cb(new AggregatedError(collectedErrors))
+      } else {
+        // emit no error
+        cb()
+      }
+    },
 
-        if (onError) {
-          try {
-            onError(err, chunk)
-          } catch {}
-        }
+    async transform(this: Transform, chunk: IN, _encoding, cb) {
+      index++
+      // console.log('transform', {index})
 
-        if (errorMode === ErrorMode.THROW_IMMEDIATELY) {
-          isRejected = true
-          // Emit error immediately
-          return cb(err)
-        }
+      // Stop processing if THROW_IMMEDIATELY mode is used
+      if (isRejected && errorMode === ErrorMode.THROW_IMMEDIATELY) return cb()
 
-        if (errorMode === ErrorMode.THROW_AGGREGATED) {
-          collectedErrors.push(err as Error)
+      // It resolves when it is successfully STARTED execution.
+      // If it's queued instead - it'll wait and resolve only upon START.
+      await q.push(async () => {
+        try {
+          const currentIndex = index // because we need to pass it to 2 functions - mapper and predicate. Refers to INPUT index (since it may return multiple outputs)
+          const res = await mapper(chunk, currentIndex)
+          const passedResults = await pFilter(
+            flattenArrayOutput && Array.isArray(res) ? res : [res],
+            async r => await predicate(r, currentIndex),
+          )
+
+          passedResults.forEach(r => this.push(r))
+        } catch (err) {
+          logger.error(err)
+
+          errors++
+
+          logErrorStats(logger)
+
+          if (onError) {
+            try {
+              onError(err, chunk)
+            } catch {}
+          }
+
+          if (errorMode === ErrorMode.THROW_IMMEDIATELY) {
+            isRejected = true
+            // Emit error immediately
+            // return cb(err as Error)
+            return this.emit('error', err as Error)
+          }
+
+          if (errorMode === ErrorMode.THROW_AGGREGATED) {
+            collectedErrors.push(err as Error)
+          }
         }
+      })
 
-        // Tell input stream that we're done processing, but emit nothing to output - not error nor result
-        cb()
-      }
+      // Resolved, which means it STARTED processing
+      // This means we can take more load
+      cb()
     },
-  )
+  })
 
   function logErrorStats(logger: CommonLogger, final = false): void {
     if (!errors) return

package/src/stream/writable/writableForEach.ts

@@ -1,5 +1,5 @@
-import { AsyncMapper, _passNothingPredicate } from '@naturalcycles/js-lib'
-import { transformMap, TransformMapOptions } from '../..'
+import { AsyncMapper, _passNothingPredicate, Mapper } from '@naturalcycles/js-lib'
+import { transformMap, TransformMapOptions, transformMapSync } from '../..'
 import { WritableTyped } from '../stream.model'
 
 /**
@@ -11,3 +11,13 @@ export function writableForEach<IN = any>(
 ): WritableTyped<IN> {
   return transformMap<IN, void>(mapper, { ...opt, predicate: _passNothingPredicate })
 }
+
+/**
+ * Just an alias to transformMap that declares OUT as void.
+ */
+export function writableForEachSync<IN = any>(
+  mapper: Mapper<IN, void>,
+  opt: TransformMapOptions<IN, void> = {},
+): WritableTyped<IN> {
+  return transformMapSync<IN, void>(mapper, { ...opt, predicate: _passNothingPredicate })
+}

package/dist/stream/transform/beta/transformMap2.js

@@ -1,98 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.transformMap2 = exports.notNullishPredicate = void 0;
-const stream_1 = require("stream");
-const js_lib_1 = require("@naturalcycles/js-lib");
-const colors_1 = require("../../../colors");
-function notNullishPredicate(item) {
-    return item !== undefined && item !== null;
-}
-exports.notNullishPredicate = notNullishPredicate;
-/**
- * Like pMap, but for streams.
- * Inspired by `through2`.
- * Main feature is concurrency control (implemented via `through2-concurrent`) and convenient options.
- * Using this allows native stream .pipe() to work and use backpressure.
- *
- * Only works in objectMode (due to through2Concurrent).
- *
- * Concurrency defaults to 16.
- *
- * If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
- */
-function transformMap2(mapper, opt = {}) {
-    const { concurrency = 16, predicate = notNullishPredicate, errorMode = js_lib_1.ErrorMode.THROW_IMMEDIATELY, flattenArrayOutput, onError, beforeFinal, metric = 'stream', logger = console, } = opt;
-    let index = -1;
-    let isRejected = false;
-    let errors = 0;
-    const collectedErrors = []; // only used if errorMode == THROW_AGGREGATED
-    const q = new js_lib_1.PQueue({
-        concurrency,
-        resolveOn: 'start',
-        // debug: true,
-    });
-    return new stream_1.Transform({
-        objectMode: true,
-        async final(cb) {
-            // console.log('transformMap final', {index}, q.inFlight, q.queueSize)
-            // wait for the current inFlight jobs to complete and push their results
-            await q.onIdle();
-            logErrorStats(logger, true);
-            await beforeFinal?.(); // call beforeFinal if defined
-            if (collectedErrors.length) {
-                // emit Aggregated error
-                cb(new js_lib_1.AggregatedError(collectedErrors));
-            }
-            else {
-                // emit no error
-                cb();
-            }
-        },
-        async transform(chunk, _encoding, cb) {
-            index++;
-            // console.log('transform', {index})
-            // Stop processing if THROW_IMMEDIATELY mode is used
-            if (isRejected && errorMode === js_lib_1.ErrorMode.THROW_IMMEDIATELY)
-                return cb();
-            // It resolves when it is successfully STARTED execution.
-            // If it's queued instead - it'll wait and resolve only upon START.
-            await q.push(async () => {
-                try {
-                    const currentIndex = index; // because we need to pass it to 2 functions - mapper and predicate. Refers to INPUT index (since it may return multiple outputs)
-                    const res = await mapper(chunk, currentIndex);
-                    const passedResults = await (0, js_lib_1.pFilter)(flattenArrayOutput && Array.isArray(res) ? res : [res], async (r) => await predicate(r, currentIndex));
-                    passedResults.forEach(r => this.push(r));
-                }
-                catch (err) {
-                    logger.error(err);
-                    errors++;
-                    logErrorStats(logger);
-                    if (onError) {
-                        try {
-                            onError(err, chunk);
-                        }
-                        catch { }
-                    }
-                    if (errorMode === js_lib_1.ErrorMode.THROW_IMMEDIATELY) {
-                        isRejected = true;
-                        // Emit error immediately
-                        // return cb(err as Error)
-                        return this.emit('error', err);
-                    }
-                    if (errorMode === js_lib_1.ErrorMode.THROW_AGGREGATED) {
-                        collectedErrors.push(err);
-                    }
-                }
-            });
-            // Resolved, which means it STARTED processing
-            // This means we can take more load
-            cb();
-        },
-    });
-    function logErrorStats(logger, final = false) {
-        if (!errors)
-            return;
-        logger.log(`${metric} ${final ? 'final ' : ''}errors: ${(0, colors_1.yellow)(errors)}`);
-    }
-}
-exports.transformMap2 = transformMap2;

package/src/stream/transform/beta/transformMap2.ts

@@ -1,134 +0,0 @@
-import { Transform } from 'stream'
-import {
-  AggregatedError,
-  AsyncMapper,
-  CommonLogger,
-  ErrorMode,
-  pFilter,
-  PQueue,
-} from '@naturalcycles/js-lib'
-import { yellow } from '../../../colors'
-import { TransformTyped } from '../../stream.model'
-import { TransformMapOptions } from '../transformMap'
-
-export function notNullishPredicate(item: any): boolean {
-  return item !== undefined && item !== null
-}
-
-/**
- * Like pMap, but for streams.
- * Inspired by `through2`.
- * Main feature is concurrency control (implemented via `through2-concurrent`) and convenient options.
- * Using this allows native stream .pipe() to work and use backpressure.
- *
- * Only works in objectMode (due to through2Concurrent).
- *
- * Concurrency defaults to 16.
- *
- * If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
- */
-export function transformMap2<IN = any, OUT = IN>(
-  mapper: AsyncMapper<IN, OUT>,
-  opt: TransformMapOptions<IN, OUT> = {},
-): TransformTyped<IN, OUT> {
-  const {
-    concurrency = 16,
-    predicate = notNullishPredicate,
-    errorMode = ErrorMode.THROW_IMMEDIATELY,
-    flattenArrayOutput,
-    onError,
-    beforeFinal,
-    metric = 'stream',
-    logger = console,
-  } = opt
-
-  let index = -1
-  let isRejected = false
-  let errors = 0
-  const collectedErrors: Error[] = [] // only used if errorMode == THROW_AGGREGATED
-
-  const q = new PQueue({
-    concurrency,
-    resolveOn: 'start',
-    // debug: true,
-  })
-
-  return new Transform({
-    objectMode: true,
-
-    async final(cb) {
-      // console.log('transformMap final', {index}, q.inFlight, q.queueSize)
-
-      // wait for the current inFlight jobs to complete and push their results
-      await q.onIdle()
-
-      logErrorStats(logger, true)
-
-      await beforeFinal?.() // call beforeFinal if defined
-
-      if (collectedErrors.length) {
-        // emit Aggregated error
-        cb(new AggregatedError(collectedErrors))
-      } else {
-        // emit no error
-        cb()
-      }
-    },
-
-    async transform(this: Transform, chunk: IN, _encoding, cb) {
-      index++
-      // console.log('transform', {index})
-
-      // Stop processing if THROW_IMMEDIATELY mode is used
-      if (isRejected && errorMode === ErrorMode.THROW_IMMEDIATELY) return cb()
-
-      // It resolves when it is successfully STARTED execution.
-      // If it's queued instead - it'll wait and resolve only upon START.
-      await q.push(async () => {
-        try {
-          const currentIndex = index // because we need to pass it to 2 functions - mapper and predicate. Refers to INPUT index (since it may return multiple outputs)
-          const res = await mapper(chunk, currentIndex)
-          const passedResults = await pFilter(
-            flattenArrayOutput && Array.isArray(res) ? res : [res],
-            async r => await predicate(r, currentIndex),
-          )
-
-          passedResults.forEach(r => this.push(r))
-        } catch (err) {
-          logger.error(err)
-
-          errors++
-
-          logErrorStats(logger)
-
-          if (onError) {
-            try {
-              onError(err, chunk)
-            } catch {}
-          }
-
-          if (errorMode === ErrorMode.THROW_IMMEDIATELY) {
-            isRejected = true
-            // Emit error immediately
-            // return cb(err as Error)
-            return this.emit('error', err as Error)
-          }
-
-          if (errorMode === ErrorMode.THROW_AGGREGATED) {
-            collectedErrors.push(err as Error)
-          }
-        }
-      })
-
-      // Resolved, which means it STARTED processing
-      // This means we can take more load
-      cb()
-    },
-  })
-
-  function logErrorStats(logger: CommonLogger, final = false): void {
-    if (!errors) return
-
-    logger.log(`${metric} ${final ? 'final ' : ''}errors: ${yellow(errors)}`)
-  }
-}