@naturalcycles/nodejs-lib 12.59.0 → 12.62.1

package/dist/got/getGot.js

@@ -4,7 +4,6 @@ exports.getGot = void 0;
 const js_lib_1 = require("@naturalcycles/js-lib");
 const got_1 = require("got");
 const __1 = require("..");
-const colors_1 = require("../colors");
 /**
  * Returns instance of Got with "reasonable defaults":
  *
@@ -14,12 +13,32 @@ const colors_1 = require("../colors");
  */
 function getGot(opt = {}) {
     opt.logger || (opt.logger = console);
+    if (opt.debug) {
+        opt.logStart = opt.logFinished = opt.logResponse = true;
+    }
     return got_1.default.extend({
         // Most-important is to set to anything non-empty (so, requests don't "hang" by default).
         // Should be long enough to handle for slow responses from scaled cloud APIs in times of spikes
         // Ideally should be LESS than default Request timeout in backend-lib (so, it has a chance to error
         // before server times out with 503).
-        timeout: 90000,
+        //
+        // UPD 2021-11-27
+        // There are 2 types/strategies for requests:
+        // 1. Optimized to get result no matter what. E.g in Cron jobs, where otherwise there'll be a job failure
+        // 2. Part of the Backend request, where we better retry quickly and fail on timeout before Backend aborts it with "503 Request timeout"
+        //
+        // Here it's hard to set the default timeout right for both use-cases.
+        // So, if it's important, you should override it according to your use-cases:
+        // - set it longer for Type 1 (e.g 120 seconds)
+        // - set it shorter for Type 2 (e.g 10/20 seconds)
+        // Please beware of default Retry strategy of Got:
+        // by default it will retry 2 times (after first try)
+        // First delay between tries will be ~1 second, then ~2 seconds
+        // Each retry it'll wait up to `timeout` (so, up to 60 seconds by default).
+        // So, for 3 tries it multiplies your timeout by 3 (+3 seconds between the tries).
+        // So, e.g 60 seconds timeout with 2 retries becomes up to 183 seconds.
+        // Which definitely doesn't fit into default "RequestTimeout"
+        timeout: 60000,
         ...opt,
         hooks: {
             ...opt.hooks,
@@ -33,6 +52,11 @@ function getGot(opt = {}) {
                 // User hooks go AFTER
                 ...(opt.hooks?.beforeRequest || []),
             ],
+            beforeRetry: [
+                gotBeforeRetryHook(opt),
+                // User hooks go AFTER
+                ...(opt.hooks?.beforeRetry || []),
+            ],
             afterResponse: [
                 ...(opt.hooks?.afterResponse || []),
                 // User hooks go BEFORE
@@ -69,20 +93,33 @@ exports.getGot = getGot;
 function gotErrorHook(opt = {}) {
     const { maxResponseLength = 10000 } = opt;
     return err => {
-        if (err instanceof got_1.HTTPError) {
-            const { statusCode } = err.response;
-            const { method, url, prefixUrl } = err.options;
-            const shortUrl = getShortUrl(opt, url, prefixUrl);
-            // const { started } = context as GotRequestContext
-            const body = (0, __1.inspectAny)(err.response.body, {
+        const statusCode = err.response?.statusCode || 0;
+        const { method, url, prefixUrl } = err.options;
+        const shortUrl = getShortUrl(opt, url, prefixUrl);
+        const { started, retryCount } = (err.request?.options.context || {});
+        const body = err.response?.body
+            ? (0, __1.inspectAny)(err.response.body, {
                 maxLen: maxResponseLength,
                 colors: false,
-            });
-            // timings are not part of err.message to allow automatic error grouping in Sentry
-            err.message = [[statusCode, method, shortUrl].filter(Boolean).join(' '), body]
-                .filter(Boolean)
-                .join('\n');
-        }
+            })
+            : err.message;
+        // We don't include Response/Body/Message in the log, because it's included in the Error thrown from here
+        opt.logger.log([
+            ' <<',
+            statusCode,
+            method,
+            shortUrl,
+            retryCount && `(retry ${retryCount})`,
+            'error',
+            started && 'in ' + (0, js_lib_1._since)(started),
+        ]
+            .filter(Boolean)
+            .join(' '));
+        // timings are not part of err.message to allow automatic error grouping in Sentry
+        // Colors are not used, because there's high chance that this Error will be propagated all the way to the Frontend
+        err.message = [[statusCode, method, shortUrl].filter(Boolean).join(' '), body]
+            .filter(Boolean)
+            .join('\n');
         return err;
     };
 }
@@ -93,24 +130,64 @@ function gotBeforeRequestHook(opt) {
             started: Date.now(),
         };
         if (opt.logStart) {
+            const { retryCount } = options.context;
             const shortUrl = getShortUrl(opt, options.url, options.prefixUrl);
-            opt.logger.log([(0, colors_1.dimGrey)(' >>'), (0, colors_1.dimGrey)(options.method), (0, colors_1.grey)(shortUrl)].join(' '));
+            opt.logger.log([' >>', options.method, shortUrl, retryCount && `(retry ${retryCount})`].join(' '));
         }
     };
 }
+// Here we log always, because it's similar to ErrorHook - we always log errors
+// Because Retries are always result of some Error
+function gotBeforeRetryHook(opt) {
+    const { maxResponseLength = 10000 } = opt;
+    return (options, err, retryCount) => {
+        // opt.logger!.log('beforeRetry', retryCount)
+        const statusCode = err?.response?.statusCode || 0;
+        const { method, url, prefixUrl } = options;
+        const shortUrl = getShortUrl(opt, url, prefixUrl);
+        const { started } = options.context;
+        Object.assign(options.context, { retryCount });
+        const body = err?.response?.body
+            ? (0, __1.inspectAny)(err.response.body, {
+                maxLen: maxResponseLength,
+                colors: false,
+            })
+            : err?.message;
+        // We don't include Response/Body/Message in the log, because it's included in the Error thrown from here
+        opt.logger.warn([
+            [
+                ' <<',
+                statusCode,
+                method,
+                shortUrl,
+                retryCount && retryCount > 1 ? `(retry ${retryCount - 1})` : '(first try)',
+                'error',
+                started && 'in ' + (0, js_lib_1._since)(started),
+            ]
+                .filter(Boolean)
+                .join(' '),
+            body,
+        ]
+            .filter(Boolean)
+            .join('\n'));
+    };
+}
+// AfterResponseHook is never called on Error
+// So, coloredHttpCode(resp.statusCode) is probably useless
 function gotAfterResponseHook(opt = {}) {
     return resp => {
         const success = resp.statusCode >= 200 && resp.statusCode < 400;
         if (opt.logFinished) {
-            const { started } = resp.request.options.context;
+            const { started, retryCount } = resp.request.options.context;
             const { url, prefixUrl, method } = resp.request.options;
             const shortUrl = getShortUrl(opt, url, prefixUrl);
             opt.logger.log([
-                (0, colors_1.dimGrey)(' <<'),
-                coloredHttpCode(resp.statusCode),
-                (0, colors_1.dimGrey)(method),
-                (0, colors_1.grey)(shortUrl),
-                started && (0, colors_1.dimGrey)('in ' + (0, js_lib_1._since)(started)),
+                ' <<',
+                resp.statusCode,
+                method,
+                shortUrl,
+                retryCount && `(retry ${retryCount - 1})`,
+                started && 'in ' + (0, js_lib_1._since)(started),
             ]
                 .filter(Boolean)
                 .join(' '));
@@ -123,13 +200,6 @@ function gotAfterResponseHook(opt = {}) {
         return resp;
     };
 }
-function coloredHttpCode(statusCode) {
-    if (statusCode < 400)
-        return (0, colors_1.dimGrey)(statusCode); // default
-    if (statusCode < 500)
-        return (0, colors_1.yellow)(statusCode);
-    return (0, colors_1.red)(statusCode);
-}
 function getShortUrl(opt, url, prefixUrl) {
     let shortUrl = url.toString();
     if (opt.logWithSearchParams === false) {

package/dist/got/got.model.d.ts

@@ -1,6 +1,11 @@
 import { AnyObject, CommonLogger } from '@naturalcycles/js-lib';
 import type { Options } from 'got';
 export interface GetGotOptions extends Options {
+    /**
+     * Set to `true` to enable all possible debug logging.
+     * Not safe in prod (as it logs Responses), but great to use during development.
+     */
+    debug?: boolean;
     /**
      * @default false
      */
@@ -43,4 +48,5 @@ export interface GotRequestContext extends AnyObject {
      * Millisecond-timestamp of when the request was started. To be able to count "time spent".
      */
     started: number;
+    retryCount?: number;
 }

package/dist/index.d.ts

@@ -1,5 +1,5 @@
 import Ajv from 'ajv';
-import { HTTPError, TimeoutError } from 'got';
+import { RequestError, TimeoutError } from 'got';
 import type { AfterResponseHook, BeforeErrorHook, BeforeRequestHook, Got } from 'got';
 import { AnySchema, ValidationErrorItem } from 'joi';
 import { _chunkBuffer } from './buffer/buffer.util';
@@ -11,14 +11,14 @@ import { Debug, IDebug, IDebugger } from './log/debug';
 export * from './security/hash.util';
 export * from './security/id.util';
 export * from './security/secret.util';
-import { hasColors } from './colors/colors';
+export * from './colors/colors';
 export * from './log/log.util';
 import { slackDefaultMessagePrefixHook, SlackService } from './slack/slack.service';
 import { SlackApiBody, SlackMessage, SlackMessagePrefixHook, SlackMessageProps, SlackServiceCfg } from './slack/slack.service.model';
 import { NDJsonStats } from './stream/ndjson/ndjson.model';
-import { ndJsonFileRead } from './stream/ndjson/ndJsonFileRead';
-import { ndJsonFileWrite } from './stream/ndjson/ndJsonFileWrite';
-import { ndjsonMap } from './stream/ndjson/ndjsonMap';
+export * from './stream/ndjson/ndJsonFileRead';
+export * from './stream/ndjson/ndJsonFileWrite';
+export * from './stream/ndjson/ndjsonMap';
 import { ndjsonStreamForEach, NDJSONStreamForEachOptions } from './stream/ndjson/ndjsonStreamForEach';
 import { pipelineFromNDJsonFile, PipelineFromNDJsonFileOptions } from './stream/ndjson/pipelineFromNDJsonFile';
 import { pipelineToNDJsonFile, PipelineToNDJsonFileOptions } from './stream/ndjson/pipelineToNDJsonFile';
@@ -26,25 +26,25 @@ import { streamToNDJsonFile } from './stream/ndjson/streamToNDJsonFile';
 import { bufferReviver, transformJsonParse, TransformJsonParseOptions } from './stream/ndjson/transformJsonParse';
 import { transformToNDJson, TransformToNDJsonOptions } from './stream/ndjson/transformToNDJson';
 export * from './stream/pipeline/pipeline';
-import { readableCreate, readableFrom } from './stream/readable/readableCreate';
-import { readableForEach, readableForEachSync } from './stream/readable/readableForEach';
-import { readableFromArray } from './stream/readable/readableFromArray';
-import { readableMap } from './stream/readable/readableMap';
-import { readableMapToArray } from './stream/readable/readableMapToArray';
-import { readableToArray } from './stream/readable/readableToArray';
+export * from './stream/readable/readableCreate';
+export * from './stream/readable/readableForEach';
+export * from './stream/readable/readableFromArray';
+export * from './stream/readable/readableMap';
+export * from './stream/readable/readableMapToArray';
+export * from './stream/readable/readableToArray';
 import { ReadableTyped, TransformOptions, TransformTyped, WritableTyped } from './stream/stream.model';
 export * from './stream/transform/transformBuffer';
 export * from './stream/transform/transformFilter';
 export * from './stream/transform/transformLimit';
 export * from './stream/transform/transformLogProgress';
 import { transformMap, TransformMapOptions } from './stream/transform/transformMap';
-import { transformMapSimple } from './stream/transform/transformMapSimple';
-import { transformNoOp } from './stream/transform/transformNoOp';
+export * from './stream/transform/transformMapSimple';
+export * from './stream/transform/transformNoOp';
 import { transformMapSync, TransformMapSyncOptions } from './stream/transform/transformMapSync';
-import { transformSplit } from './stream/transform/transformSplit';
-import { transformTap } from './stream/transform/transformTap';
-import { transformToArray } from './stream/transform/transformToArray';
-import { transformToString } from './stream/transform/transformToString';
+export * from './stream/transform/transformSplit';
+export * from './stream/transform/transformTap';
+export * from './stream/transform/transformToArray';
+export * from './stream/transform/transformToString';
 import { BaseWorkerClass, WorkerClassInterface } from './stream/transform/worker/baseWorkerClass';
 import { transformMultiThreaded, TransformMultiThreadedOptions } from './stream/transform/worker/transformMultiThreaded';
 import { WorkerInput, WorkerOutput } from './stream/transform/worker/transformMultiThreaded.model';
@@ -59,7 +59,7 @@ export * from './util/zip.util';
 import { readAjvSchemas, readJsonSchemas } from './validation/ajv/ajv.util';
 import { AjvSchema, AjvSchemaCfg, AjvValidationOptions } from './validation/ajv/ajvSchema';
 import { AjvValidationError, AjvValidationErrorData } from './validation/ajv/ajvValidationError';
-import { getAjv } from './validation/ajv/getAjv';
+export * from './validation/ajv/getAjv';
 import { ExtendedJoi, Joi } from './validation/joi/joi.extensions';
 import { AnySchemaTyped, ArraySchemaTyped, BooleanSchemaTyped, NumberSchemaTyped, ObjectSchemaTyped, SchemaTyped, StringSchemaTyped } from './validation/joi/joi.model';
 export * from './validation/joi/joi.shared.schemas';
@@ -68,4 +68,4 @@ import { convert, getValidationResult, isValid, JoiValidationResult, undefinedIf
 import { sanitizeHTML, SanitizeHTMLOptions } from './validation/sanitize.util';
 import { runScript, RunScriptOptions } from './script';
 export type { RunScriptOptions, JoiValidationErrorData, JoiValidationResult, ValidationErrorItem, ExtendedJoi, SchemaTyped, AnySchema, AnySchemaTyped, ArraySchemaTyped, BooleanSchemaTyped, NumberSchemaTyped, ObjectSchemaTyped, StringSchemaTyped, IDebug, IDebugger, SlackServiceCfg, SlackMessage, SlackMessageProps, SlackApiBody, SlackMessagePrefixHook, ReadableTyped, WritableTyped, TransformTyped, PipelineFromNDJsonFileOptions, PipelineToNDJsonFileOptions, TransformJsonParseOptions, TransformToNDJsonOptions, TransformMapOptions, TransformMapSyncOptions, NDJSONStreamForEachOptions, TransformOptions, TransformMultiThreadedOptions, WorkerClassInterface, WorkerInput, WorkerOutput, TableDiffOptions, InspectAnyOptions, Got, GetGotOptions, AfterResponseHook, BeforeErrorHook, BeforeRequestHook, AjvValidationOptions, AjvSchemaCfg, AjvValidationErrorData, SanitizeHTMLOptions, };
-export { JoiValidationError, validate, getValidationResult, isValid, undefinedIfInvalid, convert, Joi, LRUMemoCache, Debug, SlackService, slackDefaultMessagePrefixHook, readableCreate, readableFrom, readableFromArray, readableToArray, readableForEach, readableForEachSync, readableMap, readableMapToArray, ndjsonMap, ndJsonFileRead, ndJsonFileWrite, ndjsonStreamForEach, pipelineFromNDJsonFile, pipelineToNDJsonFile, NDJsonStats, streamToNDJsonFile, transformJsonParse, bufferReviver, transformToNDJson, transformMap, transformMapSync, transformMapSimple, transformNoOp, transformSplit, transformToString, transformToArray, transformTap, transformMultiThreaded, BaseWorkerClass, tableDiff, inspectAny, inspectAnyStringifyFn, HTTPError, TimeoutError, _chunkBuffer, Ajv, getAjv, AjvSchema, AjvValidationError, readJsonSchemas, readAjvSchemas, hasColors, sanitizeHTML, runScript, };
+export { JoiValidationError, validate, getValidationResult, isValid, undefinedIfInvalid, convert, Joi, LRUMemoCache, Debug, SlackService, slackDefaultMessagePrefixHook, ndjsonStreamForEach, pipelineFromNDJsonFile, pipelineToNDJsonFile, NDJsonStats, streamToNDJsonFile, transformJsonParse, bufferReviver, transformToNDJson, transformMap, transformMapSync, transformMultiThreaded, BaseWorkerClass, tableDiff, inspectAny, inspectAnyStringifyFn, RequestError, TimeoutError, _chunkBuffer, Ajv, AjvSchema, AjvValidationError, readJsonSchemas, readAjvSchemas, sanitizeHTML, runScript, };

package/dist/index.js

@@ -1,12 +1,11 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.AjvValidationError = exports.AjvSchema = exports.getAjv = exports.Ajv = exports._chunkBuffer = exports.TimeoutError = exports.HTTPError = exports.inspectAnyStringifyFn = exports.inspectAny = exports.tableDiff = exports.BaseWorkerClass = exports.transformMultiThreaded = exports.transformTap = exports.transformToArray = exports.transformToString = exports.transformSplit = exports.transformNoOp = exports.transformMapSimple = exports.transformMapSync = exports.transformMap = exports.transformToNDJson = exports.bufferReviver = exports.transformJsonParse = exports.streamToNDJsonFile = exports.NDJsonStats = exports.pipelineToNDJsonFile = exports.pipelineFromNDJsonFile = exports.ndjsonStreamForEach = exports.ndJsonFileWrite = exports.ndJsonFileRead = exports.ndjsonMap = exports.readableMapToArray = exports.readableMap = exports.readableForEachSync = exports.readableForEach = exports.readableToArray = exports.readableFromArray = exports.readableFrom = exports.readableCreate = exports.slackDefaultMessagePrefixHook = exports.SlackService = exports.Debug = exports.LRUMemoCache = exports.Joi = exports.convert = exports.undefinedIfInvalid = exports.isValid = exports.getValidationResult = exports.validate = exports.JoiValidationError = void 0;
-exports.runScript = exports.sanitizeHTML = exports.hasColors = exports.readAjvSchemas = exports.readJsonSchemas = void 0;
+exports.runScript = exports.sanitizeHTML = exports.readAjvSchemas = exports.readJsonSchemas = exports.AjvValidationError = exports.AjvSchema = exports.Ajv = exports._chunkBuffer = exports.TimeoutError = exports.RequestError = exports.inspectAnyStringifyFn = exports.inspectAny = exports.tableDiff = exports.BaseWorkerClass = exports.transformMultiThreaded = exports.transformMapSync = exports.transformMap = exports.transformToNDJson = exports.bufferReviver = exports.transformJsonParse = exports.streamToNDJsonFile = exports.NDJsonStats = exports.pipelineToNDJsonFile = exports.pipelineFromNDJsonFile = exports.ndjsonStreamForEach = exports.slackDefaultMessagePrefixHook = exports.SlackService = exports.Debug = exports.LRUMemoCache = exports.Joi = exports.convert = exports.undefinedIfInvalid = exports.isValid = exports.getValidationResult = exports.validate = exports.JoiValidationError = void 0;
 const tslib_1 = require("tslib");
 const ajv_1 = require("ajv");
 exports.Ajv = ajv_1.default;
 const got_1 = require("got");
-Object.defineProperty(exports, "HTTPError", { enumerable: true, get: function () { return got_1.HTTPError; } });
+Object.defineProperty(exports, "RequestError", { enumerable: true, get: function () { return got_1.RequestError; } });
 Object.defineProperty(exports, "TimeoutError", { enumerable: true, get: function () { return got_1.TimeoutError; } });
 const buffer_util_1 = require("./buffer/buffer.util");
 Object.defineProperty(exports, "_chunkBuffer", { enumerable: true, get: function () { return buffer_util_1._chunkBuffer; } });
@@ -19,20 +18,16 @@ Object.defineProperty(exports, "Debug", { enumerable: true, get: function () { r
 (0, tslib_1.__exportStar)(require("./security/hash.util"), exports);
 (0, tslib_1.__exportStar)(require("./security/id.util"), exports);
 (0, tslib_1.__exportStar)(require("./security/secret.util"), exports);
-const colors_1 = require("./colors/colors");
-Object.defineProperty(exports, "hasColors", { enumerable: true, get: function () { return colors_1.hasColors; } });
+(0, tslib_1.__exportStar)(require("./colors/colors"), exports);
 (0, tslib_1.__exportStar)(require("./log/log.util"), exports);
 const slack_service_1 = require("./slack/slack.service");
 Object.defineProperty(exports, "slackDefaultMessagePrefixHook", { enumerable: true, get: function () { return slack_service_1.slackDefaultMessagePrefixHook; } });
 Object.defineProperty(exports, "SlackService", { enumerable: true, get: function () { return slack_service_1.SlackService; } });
 const ndjson_model_1 = require("./stream/ndjson/ndjson.model");
 Object.defineProperty(exports, "NDJsonStats", { enumerable: true, get: function () { return ndjson_model_1.NDJsonStats; } });
-const ndJsonFileRead_1 = require("./stream/ndjson/ndJsonFileRead");
-Object.defineProperty(exports, "ndJsonFileRead", { enumerable: true, get: function () { return ndJsonFileRead_1.ndJsonFileRead; } });
-const ndJsonFileWrite_1 = require("./stream/ndjson/ndJsonFileWrite");
-Object.defineProperty(exports, "ndJsonFileWrite", { enumerable: true, get: function () { return ndJsonFileWrite_1.ndJsonFileWrite; } });
-const ndjsonMap_1 = require("./stream/ndjson/ndjsonMap");
-Object.defineProperty(exports, "ndjsonMap", { enumerable: true, get: function () { return ndjsonMap_1.ndjsonMap; } });
+(0, tslib_1.__exportStar)(require("./stream/ndjson/ndJsonFileRead"), exports);
+(0, tslib_1.__exportStar)(require("./stream/ndjson/ndJsonFileWrite"), exports);
+(0, tslib_1.__exportStar)(require("./stream/ndjson/ndjsonMap"), exports);
 const ndjsonStreamForEach_1 = require("./stream/ndjson/ndjsonStreamForEach");
 Object.defineProperty(exports, "ndjsonStreamForEach", { enumerable: true, get: function () { return ndjsonStreamForEach_1.ndjsonStreamForEach; } });
 const pipelineFromNDJsonFile_1 = require("./stream/ndjson/pipelineFromNDJsonFile");
@@ -47,40 +42,26 @@ Object.defineProperty(exports, "transformJsonParse", { enumerable: true, get: fu
 const transformToNDJson_1 = require("./stream/ndjson/transformToNDJson");
 Object.defineProperty(exports, "transformToNDJson", { enumerable: true, get: function () { return transformToNDJson_1.transformToNDJson; } });
 (0, tslib_1.__exportStar)(require("./stream/pipeline/pipeline"), exports);
-const readableCreate_1 = require("./stream/readable/readableCreate");
-Object.defineProperty(exports, "readableCreate", { enumerable: true, get: function () { return readableCreate_1.readableCreate; } });
-Object.defineProperty(exports, "readableFrom", { enumerable: true, get: function () { return readableCreate_1.readableFrom; } });
-const readableForEach_1 = require("./stream/readable/readableForEach");
-Object.defineProperty(exports, "readableForEach", { enumerable: true, get: function () { return readableForEach_1.readableForEach; } });
-Object.defineProperty(exports, "readableForEachSync", { enumerable: true, get: function () { return readableForEach_1.readableForEachSync; } });
-const readableFromArray_1 = require("./stream/readable/readableFromArray");
-Object.defineProperty(exports, "readableFromArray", { enumerable: true, get: function () { return readableFromArray_1.readableFromArray; } });
-const readableMap_1 = require("./stream/readable/readableMap");
-Object.defineProperty(exports, "readableMap", { enumerable: true, get: function () { return readableMap_1.readableMap; } });
-const readableMapToArray_1 = require("./stream/readable/readableMapToArray");
-Object.defineProperty(exports, "readableMapToArray", { enumerable: true, get: function () { return readableMapToArray_1.readableMapToArray; } });
-const readableToArray_1 = require("./stream/readable/readableToArray");
-Object.defineProperty(exports, "readableToArray", { enumerable: true, get: function () { return readableToArray_1.readableToArray; } });
+(0, tslib_1.__exportStar)(require("./stream/readable/readableCreate"), exports);
+(0, tslib_1.__exportStar)(require("./stream/readable/readableForEach"), exports);
+(0, tslib_1.__exportStar)(require("./stream/readable/readableFromArray"), exports);
+(0, tslib_1.__exportStar)(require("./stream/readable/readableMap"), exports);
+(0, tslib_1.__exportStar)(require("./stream/readable/readableMapToArray"), exports);
+(0, tslib_1.__exportStar)(require("./stream/readable/readableToArray"), exports);
 (0, tslib_1.__exportStar)(require("./stream/transform/transformBuffer"), exports);
 (0, tslib_1.__exportStar)(require("./stream/transform/transformFilter"), exports);
 (0, tslib_1.__exportStar)(require("./stream/transform/transformLimit"), exports);
 (0, tslib_1.__exportStar)(require("./stream/transform/transformLogProgress"), exports);
 const transformMap_1 = require("./stream/transform/transformMap");
 Object.defineProperty(exports, "transformMap", { enumerable: true, get: function () { return transformMap_1.transformMap; } });
-const transformMapSimple_1 = require("./stream/transform/transformMapSimple");
-Object.defineProperty(exports, "transformMapSimple", { enumerable: true, get: function () { return transformMapSimple_1.transformMapSimple; } });
-const transformNoOp_1 = require("./stream/transform/transformNoOp");
-Object.defineProperty(exports, "transformNoOp", { enumerable: true, get: function () { return transformNoOp_1.transformNoOp; } });
+(0, tslib_1.__exportStar)(require("./stream/transform/transformMapSimple"), exports);
+(0, tslib_1.__exportStar)(require("./stream/transform/transformNoOp"), exports);
 const transformMapSync_1 = require("./stream/transform/transformMapSync");
 Object.defineProperty(exports, "transformMapSync", { enumerable: true, get: function () { return transformMapSync_1.transformMapSync; } });
-const transformSplit_1 = require("./stream/transform/transformSplit");
-Object.defineProperty(exports, "transformSplit", { enumerable: true, get: function () { return transformSplit_1.transformSplit; } });
-const transformTap_1 = require("./stream/transform/transformTap");
-Object.defineProperty(exports, "transformTap", { enumerable: true, get: function () { return transformTap_1.transformTap; } });
-const transformToArray_1 = require("./stream/transform/transformToArray");
-Object.defineProperty(exports, "transformToArray", { enumerable: true, get: function () { return transformToArray_1.transformToArray; } });
-const transformToString_1 = require("./stream/transform/transformToString");
-Object.defineProperty(exports, "transformToString", { enumerable: true, get: function () { return transformToString_1.transformToString; } });
+(0, tslib_1.__exportStar)(require("./stream/transform/transformSplit"), exports);
+(0, tslib_1.__exportStar)(require("./stream/transform/transformTap"), exports);
+(0, tslib_1.__exportStar)(require("./stream/transform/transformToArray"), exports);
+(0, tslib_1.__exportStar)(require("./stream/transform/transformToString"), exports);
 const baseWorkerClass_1 = require("./stream/transform/worker/baseWorkerClass");
 Object.defineProperty(exports, "BaseWorkerClass", { enumerable: true, get: function () { return baseWorkerClass_1.BaseWorkerClass; } });
 const transformMultiThreaded_1 = require("./stream/transform/worker/transformMultiThreaded");
@@ -103,8 +84,7 @@ const ajvSchema_1 = require("./validation/ajv/ajvSchema");
 Object.defineProperty(exports, "AjvSchema", { enumerable: true, get: function () { return ajvSchema_1.AjvSchema; } });
 const ajvValidationError_1 = require("./validation/ajv/ajvValidationError");
 Object.defineProperty(exports, "AjvValidationError", { enumerable: true, get: function () { return ajvValidationError_1.AjvValidationError; } });
-const getAjv_1 = require("./validation/ajv/getAjv");
-Object.defineProperty(exports, "getAjv", { enumerable: true, get: function () { return getAjv_1.getAjv; } });
+(0, tslib_1.__exportStar)(require("./validation/ajv/getAjv"), exports);
 const joi_extensions_1 = require("./validation/joi/joi.extensions");
 Object.defineProperty(exports, "Joi", { enumerable: true, get: function () { return joi_extensions_1.Joi; } });
 (0, tslib_1.__exportStar)(require("./validation/joi/joi.shared.schemas"), exports);

package/dist/stream/ndjson/transformJsonParse.js

@@ -19,19 +19,19 @@ const stream_1 = require("stream");
 function transformJsonParse(opt = {}) {
     const { strict = true, reviver } = opt;
     return new stream_1.Transform({
-        objectMode: false,
+        writableObjectMode: false,
         readableObjectMode: true,
-        transform(chunk, _encoding, cb) {
+        transform(chunk, _, cb) {
             try {
                 const data = JSON.parse(chunk, reviver);
                 cb(null, data);
             }
             catch (err) {
-                // console.error(err)
                 if (strict) {
                     cb(err); // emit error
                 }
                 else {
+                    console.error(err);
                     cb(); // emit no error, but no result neither
                 }
             }

package/dist/stream/ndjson/transformToNDJson.js

@@ -9,9 +9,9 @@ const js_lib_1 = require("@naturalcycles/js-lib");
 function transformToNDJson(opt = {}) {
     const { strict = true, separator = '\n', sortObjects = false, useFlatstr = false } = opt;
     return new stream_1.Transform({
-        objectMode: true,
+        writableObjectMode: true,
         readableObjectMode: false,
-        transform(chunk, _encoding, cb) {
+        transform(chunk, _, cb) {
             try {
                 if (sortObjects) {
                     chunk = (0, js_lib_1._sortObjectDeep)(chunk);

package/dist/stream/sizeStack.d.ts

@@ -0,0 +1,9 @@
+import { CommonLogger, NumberStack } from '@naturalcycles/js-lib';
+export declare class SizeStack extends NumberStack {
+    name: string;
+    constructor(name: string, size: number);
+    total: number;
+    push(item: any): this;
+    getStats(): string;
+    static countItem(item: any, logger: CommonLogger, sizes?: SizeStack, sizesZipped?: SizeStack): Promise<void>;
+}

package/dist/stream/sizeStack.js

@@ -0,0 +1,48 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SizeStack = void 0;
+const js_lib_1 = require("@naturalcycles/js-lib");
+const colors_1 = require("../colors");
+const zip_util_1 = require("../util/zip.util");
+class SizeStack extends js_lib_1.NumberStack {
+    constructor(name, size) {
+        super(size);
+        this.name = name;
+        this.total = 0;
+    }
+    push(item) {
+        this.total += item;
+        return super.push(item);
+    }
+    getStats() {
+        // const pcs = this.percentiles([50, 90])
+        return [
+            '  ' + this.name,
+            'avg',
+            (0, colors_1.yellow)((0, js_lib_1._hb)(this.avg())),
+            // 'p50',
+            // yellow(_hb(pcs[50])),
+            // 'p90',
+            // yellow(_hb(pcs[90])),
+            'total',
+            (0, colors_1.yellow)((0, js_lib_1._hb)(this.total)),
+        ].join(' ');
+    }
+    static async countItem(item, logger, sizes, sizesZipped) {
+        if (!sizes)
+            return;
+        // try-catch, because we don't want to fail the pipeline on logProgress
+        try {
+            const buf = Buffer.from(JSON.stringify(item));
+            sizes.push(buf.byteLength);
+            if (sizesZipped) {
+                const { byteLength } = await (0, zip_util_1.gzipBuffer)(buf);
+                sizesZipped.push(byteLength);
+            }
+        }
+        catch (err) {
+            logger.warn(`transformLogProgress failed to JSON.stringify the chunk: ${err.message}`);
+        }
+    }
+}
+exports.SizeStack = SizeStack;

package/dist/stream/transform/transformBuffer.js

@@ -13,7 +13,7 @@ function transformBuffer(opt) {
     return new stream_1.Transform({
         objectMode: true,
         ...opt,
-        transform(chunk, _encoding, cb) {
+        transform(chunk, _, cb) {
             buf.push(chunk);
             if (buf.length >= batchSize) {
                 cb(null, buf);

package/dist/stream/transform/transformFilter.d.ts

@@ -1,11 +1,10 @@
 import { AsyncPredicate, Predicate } from '@naturalcycles/js-lib';
 import { TransformOptions, TransformTyped } from '../stream.model';
+import { TransformMapOptions } from './transformMap';
 /**
- * Note, that currently it's NOT concurrent! (concurrency = 1)
- * So, it's recommended to use transformMap instead, that is both concurrent and has
- * filtering feature by default.
+ * Just a convenience wrapper around `transformMap` that has built-in predicate filtering support.
  */
-export declare function transformFilter<IN = any>(predicate: AsyncPredicate<IN>, opt?: TransformOptions): TransformTyped<IN, IN>;
+export declare function transformFilter<IN = any>(predicate: AsyncPredicate<IN>, opt?: TransformMapOptions): TransformTyped<IN, IN>;
 /**
  * Sync version of `transformFilter`
  */

package/dist/stream/transform/transformFilter.js

@@ -2,29 +2,14 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.transformFilterSync = exports.transformFilter = void 0;
 const stream_1 = require("stream");
+const transformMap_1 = require("./transformMap");
 /**
- * Note, that currently it's NOT concurrent! (concurrency = 1)
- * So, it's recommended to use transformMap instead, that is both concurrent and has
- * filtering feature by default.
+ * Just a convenience wrapper around `transformMap` that has built-in predicate filtering support.
  */
 function transformFilter(predicate, opt = {}) {
-    let index = 0;
-    return new stream_1.Transform({
-        objectMode: true,
+    return (0, transformMap_1.transformMap)(v => v, {
+        predicate,
         ...opt,
-        async transform(chunk, _encoding, cb) {
-            try {
-                if (await predicate(chunk, index++)) {
-                    cb(null, chunk); // pass through
-                }
-                else {
-                    cb(); // signal that we've finished processing, but emit no output here
-                }
-            }
-            catch (err) {
-                cb(err);
-            }
-        },
     });
 }
 exports.transformFilter = transformFilter;
@@ -36,7 +21,7 @@ function transformFilterSync(predicate, opt = {}) {
     return new stream_1.Transform({
         objectMode: true,
         ...opt,
-        async transform(chunk, _encoding, cb) {
+        transform(chunk, _, cb) {
             try {
                 if (predicate(chunk, index++)) {
                     cb(null, chunk); // pass through

package/dist/stream/transform/transformLogProgress.d.ts

@@ -84,6 +84,26 @@ export interface TransformLogProgressOptions<IN = any> extends TransformOptions
      * Defaults to 1.
      */
     batchSize?: number;
+    /**
+     * Experimental logging of item (shunk) sizes, when json-stringified.
+     *
+     * Defaults to false.
+     *
+     * @experimental
+     */
+    logSizes?: boolean;
+    /**
+     * How many last item sizes to keep in a buffer, to calculate stats (p50, p90, avg, etc).
+     * Defaults to 100_000.
+     * Cannot be Infinity.
+     */
+    logSizesBuffer?: number;
+    /**
+     * Works in addition to `logSizes`. Adds "zipped sizes".
+     *
+     * @experimental
+     */
+    logZippedSizes?: boolean;
 }
 /**
  * Pass-through transform that optionally logs progress.