@naturalcycles/nodejs-lib 12.52.0 → 12.55.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -13,6 +13,7 @@ const colors_1 = require("../colors");
13
13
  * 3. Reasonable defaults(tm), e.g non-infinite Timeout
14
14
  */
15
15
  function getGot(opt = {}) {
16
+ opt.logger || (opt.logger = console);
16
17
  return got_1.default.extend({
17
18
  // Most-important is to set to anything non-empty (so, requests don't "hang" by default).
18
19
  // Should be long enough to handle for slow responses from scaled cloud APIs in times of spikes
@@ -93,7 +94,7 @@ function gotBeforeRequestHook(opt) {
93
94
  };
94
95
  if (opt.logStart) {
95
96
  const shortUrl = getShortUrl(opt, options.url, options.prefixUrl);
96
- console.log([(0, colors_1.dimGrey)(' >>'), (0, colors_1.dimGrey)(options.method), (0, colors_1.grey)(shortUrl)].join(' '));
97
+ opt.logger.log([(0, colors_1.dimGrey)(' >>'), (0, colors_1.dimGrey)(options.method), (0, colors_1.grey)(shortUrl)].join(' '));
97
98
  }
98
99
  };
99
100
  }
@@ -104,7 +105,7 @@ function gotAfterResponseHook(opt = {}) {
104
105
  const { started } = resp.request.options.context;
105
106
  const { url, prefixUrl, method } = resp.request.options;
106
107
  const shortUrl = getShortUrl(opt, url, prefixUrl);
107
- console.log([
108
+ opt.logger.log([
108
109
  (0, colors_1.dimGrey)(' <<'),
109
110
  coloredHttpCode(resp.statusCode),
110
111
  (0, colors_1.dimGrey)(method),
@@ -117,7 +118,7 @@ function gotAfterResponseHook(opt = {}) {
117
118
  }
118
119
  // Error responses are not logged, cause they're included in Error message already
119
120
  if (opt.logResponse && success) {
120
- console.log((0, __1.inspectAny)((0, js_lib_1._jsonParseIfPossible)(resp.body), { maxLen: opt.maxResponseLength }));
121
+ opt.logger.log((0, __1.inspectAny)(resp.body, { maxLen: opt.maxResponseLength }));
121
122
  }
122
123
  return resp;
123
124
  };
@@ -1,4 +1,4 @@
1
- import { AnyObject } from '@naturalcycles/js-lib';
1
+ import { AnyObject, CommonLogger } from '@naturalcycles/js-lib';
2
2
  import type { Options } from 'got';
3
3
  export interface GetGotOptions extends Options {
4
4
  /**
@@ -27,6 +27,10 @@ export interface GetGotOptions extends Options {
27
27
  * Set to false to strip searchParams from url when logging (both success and error)
28
28
  */
29
29
  logWithSearchParams?: boolean;
30
+ /**
31
+ * Defaults to `console`
32
+ */
33
+ logger?: CommonLogger;
30
34
  /**
31
35
  * Max length of response object before it's truncated.
32
36
  *
package/dist/index.d.ts CHANGED
@@ -51,7 +51,7 @@ import { writableForEach } from './stream/writable/writableForEach';
51
51
  import { writableFork } from './stream/writable/writableFork';
52
52
  import { writablePushToArray } from './stream/writable/writablePushToArray';
53
53
  import { writableVoid } from './stream/writable/writableVoid';
54
- import { inspectAny, InspectAnyOptions } from './string/inspectAny';
54
+ import { inspectAny, InspectAnyOptions, inspectAnyStringifyFn } from './string/inspectAny';
55
55
  import { requireEnvKeys, requireFileToExist } from './util/env.util';
56
56
  import { LRUMemoCache } from './util/lruMemoCache';
57
57
  import { gunzipBuffer, gunzipToString, gzipBuffer, gzipString, unzipBuffer, unzipToString, zipBuffer, zipString } from './util/zip.util';
@@ -65,5 +65,6 @@ import { anyObjectSchema, anySchema, arraySchema, oneOfSchema, binarySchema, boo
65
65
  import { JoiValidationError, JoiValidationErrorData } from './validation/joi/joi.validation.error';
66
66
  import { convert, getValidationResult, isValid, JoiValidationResult, undefinedIfInvalid, validate } from './validation/joi/joi.validation.util';
67
67
  import { sanitizeHTML, SanitizeHTMLOptions } from './validation/sanitize.util';
68
- export type { JoiValidationErrorData, JoiValidationResult, ValidationErrorItem, ExtendedJoi, SchemaTyped, AnySchema, AnySchemaTyped, ArraySchemaTyped, BooleanSchemaTyped, NumberSchemaTyped, ObjectSchemaTyped, StringSchemaTyped, IDebug, IDebugger, SlackServiceCfg, SlackMessage, SlackMessageProps, SlackApiBody, SlackMessagePrefixHook, ReadableTyped, WritableTyped, TransformTyped, PipelineFromNDJsonFileOptions, PipelineToNDJsonFileOptions, TransformJsonParseOptions, TransformToNDJsonOptions, TransformMapOptions, TransformMapSyncOptions, NDJSONStreamForEachOptions, TransformOptions, TransformLogProgressOptions, TransformMultiThreadedOptions, WorkerClassInterface, WorkerInput, WorkerOutput, TableDiffOptions, InspectAnyOptions, Got, GetGotOptions, AfterResponseHook, BeforeErrorHook, BeforeRequestHook, AjvValidationOptions, AjvSchemaCfg, AjvValidationErrorData, SanitizeHTMLOptions, };
69
- export { JoiValidationError, validate, getValidationResult, isValid, undefinedIfInvalid, convert, Joi, booleanSchema, booleanDefaultToFalseSchema, stringSchema, numberSchema, integerSchema, percentageSchema, dateStringSchema, arraySchema, binarySchema, objectSchema, oneOfSchema, anySchema, anyObjectSchema, baseDBEntitySchema, savedDBEntitySchema, idSchema, unixTimestampSchema, verSchema, emailSchema, SEM_VER_PATTERN, semVerSchema, userAgentSchema, utcOffsetSchema, ipAddressSchema, slugSchema, urlSchema, processSharedUtil, zipBuffer, gzipBuffer, unzipBuffer, gunzipBuffer, zipString, gzipString, unzipToString, gunzipToString, requireEnvKeys, requireFileToExist, LRUMemoCache, stringId, stringIdAsync, stringIdUnsafe, ALPHABET_NUMBER, ALPHABET_LOWERCASE, ALPHABET_UPPERCASE, ALPHABET_ALPHANUMERIC_LOWERCASE, ALPHABET_ALPHANUMERIC_UPPERCASE, ALPHABET_ALPHANUMERIC, md5, hash, hashAsBuffer, md5AsBuffer, stringToBase64, base64ToString, bufferToBase64, base64ToBuffer, Debug, getSecretMap, setSecretMap, loadSecretsFromEnv, loadSecretsFromJsonFile, removeSecretsFromEnv, secret, secretOptional, memoryUsage, memoryUsageFull, SlackService, slackDefaultMessagePrefixHook, readableCreate, readableFrom, readableFromArray, readableToArray, readableForEach, readableForEachSync, readableMap, readableMapToArray, _pipeline, transformBuffer, ndjsonMap, ndJsonFileRead, ndJsonFileWrite, ndjsonStreamForEach, pipelineFromNDJsonFile, pipelineToNDJsonFile, NDJsonStats, streamToNDJsonFile, transformJsonParse, bufferReviver, transformToNDJson, transformFilter, transformFilterSync, transformMap, transformMapSync, transformMapSimple, transformNoOp, writableForEach, writablePushToArray, transformSplit, transformToString, transformToArray, transformTap, transformLogProgress, transformLimit, writableVoid, writableFork, transformMultiThreaded, BaseWorkerClass, tableDiff, inspectAny, getGot, HTTPError, TimeoutError, _chunkBuffer, Ajv, getAjv, AjvSchema, AjvValidationError, readJsonSchemas, readAjvSchemas, hasColors, sanitizeHTML, };
68
+ import { runScript, RunScriptOptions } from './script';
69
+ export type { RunScriptOptions, JoiValidationErrorData, JoiValidationResult, ValidationErrorItem, ExtendedJoi, SchemaTyped, AnySchema, AnySchemaTyped, ArraySchemaTyped, BooleanSchemaTyped, NumberSchemaTyped, ObjectSchemaTyped, StringSchemaTyped, IDebug, IDebugger, SlackServiceCfg, SlackMessage, SlackMessageProps, SlackApiBody, SlackMessagePrefixHook, ReadableTyped, WritableTyped, TransformTyped, PipelineFromNDJsonFileOptions, PipelineToNDJsonFileOptions, TransformJsonParseOptions, TransformToNDJsonOptions, TransformMapOptions, TransformMapSyncOptions, NDJSONStreamForEachOptions, TransformOptions, TransformLogProgressOptions, TransformMultiThreadedOptions, WorkerClassInterface, WorkerInput, WorkerOutput, TableDiffOptions, InspectAnyOptions, Got, GetGotOptions, AfterResponseHook, BeforeErrorHook, BeforeRequestHook, AjvValidationOptions, AjvSchemaCfg, AjvValidationErrorData, SanitizeHTMLOptions, };
70
+ export { JoiValidationError, validate, getValidationResult, isValid, undefinedIfInvalid, convert, Joi, booleanSchema, booleanDefaultToFalseSchema, stringSchema, numberSchema, integerSchema, percentageSchema, dateStringSchema, arraySchema, binarySchema, objectSchema, oneOfSchema, anySchema, anyObjectSchema, baseDBEntitySchema, savedDBEntitySchema, idSchema, unixTimestampSchema, verSchema, emailSchema, SEM_VER_PATTERN, semVerSchema, userAgentSchema, utcOffsetSchema, ipAddressSchema, slugSchema, urlSchema, processSharedUtil, zipBuffer, gzipBuffer, unzipBuffer, gunzipBuffer, zipString, gzipString, unzipToString, gunzipToString, requireEnvKeys, requireFileToExist, LRUMemoCache, stringId, stringIdAsync, stringIdUnsafe, ALPHABET_NUMBER, ALPHABET_LOWERCASE, ALPHABET_UPPERCASE, ALPHABET_ALPHANUMERIC_LOWERCASE, ALPHABET_ALPHANUMERIC_UPPERCASE, ALPHABET_ALPHANUMERIC, md5, hash, hashAsBuffer, md5AsBuffer, stringToBase64, base64ToString, bufferToBase64, base64ToBuffer, Debug, getSecretMap, setSecretMap, loadSecretsFromEnv, loadSecretsFromJsonFile, removeSecretsFromEnv, secret, secretOptional, memoryUsage, memoryUsageFull, SlackService, slackDefaultMessagePrefixHook, readableCreate, readableFrom, readableFromArray, readableToArray, readableForEach, readableForEachSync, readableMap, readableMapToArray, _pipeline, transformBuffer, ndjsonMap, ndJsonFileRead, ndJsonFileWrite, ndjsonStreamForEach, pipelineFromNDJsonFile, pipelineToNDJsonFile, NDJsonStats, streamToNDJsonFile, transformJsonParse, bufferReviver, transformToNDJson, transformFilter, transformFilterSync, transformMap, transformMapSync, transformMapSimple, transformNoOp, writableForEach, writablePushToArray, transformSplit, transformToString, transformToArray, transformTap, transformLogProgress, transformLimit, writableVoid, writableFork, transformMultiThreaded, BaseWorkerClass, tableDiff, inspectAny, inspectAnyStringifyFn, getGot, HTTPError, TimeoutError, _chunkBuffer, Ajv, getAjv, AjvSchema, AjvValidationError, readJsonSchemas, readAjvSchemas, hasColors, sanitizeHTML, runScript, };
package/dist/index.js CHANGED
@@ -2,7 +2,7 @@
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.ALPHABET_LOWERCASE = exports.ALPHABET_NUMBER = exports.stringIdUnsafe = exports.stringIdAsync = exports.stringId = exports.LRUMemoCache = exports.requireFileToExist = exports.requireEnvKeys = exports.gunzipToString = exports.unzipToString = exports.gzipString = exports.zipString = exports.gunzipBuffer = exports.unzipBuffer = exports.gzipBuffer = exports.zipBuffer = exports.processSharedUtil = exports.urlSchema = exports.slugSchema = exports.ipAddressSchema = exports.utcOffsetSchema = exports.userAgentSchema = exports.semVerSchema = exports.SEM_VER_PATTERN = exports.emailSchema = exports.verSchema = exports.unixTimestampSchema = exports.idSchema = exports.savedDBEntitySchema = exports.baseDBEntitySchema = exports.anyObjectSchema = exports.anySchema = exports.oneOfSchema = exports.objectSchema = exports.binarySchema = exports.arraySchema = exports.dateStringSchema = exports.percentageSchema = exports.integerSchema = exports.numberSchema = exports.stringSchema = exports.booleanDefaultToFalseSchema = exports.booleanSchema = exports.Joi = exports.convert = exports.undefinedIfInvalid = exports.isValid = exports.getValidationResult = exports.validate = exports.JoiValidationError = void 0;
4
4
  exports.transformMapSimple = exports.transformMapSync = exports.transformMap = exports.transformFilterSync = exports.transformFilter = exports.transformToNDJson = exports.bufferReviver = exports.transformJsonParse = exports.streamToNDJsonFile = exports.NDJsonStats = exports.pipelineToNDJsonFile = exports.pipelineFromNDJsonFile = exports.ndjsonStreamForEach = exports.ndJsonFileWrite = exports.ndJsonFileRead = exports.ndjsonMap = exports.transformBuffer = exports._pipeline = exports.readableMapToArray = exports.readableMap = exports.readableForEachSync = exports.readableForEach = exports.readableToArray = exports.readableFromArray = exports.readableFrom = exports.readableCreate = exports.slackDefaultMessagePrefixHook = exports.SlackService = exports.memoryUsageFull = exports.memoryUsage = exports.secretOptional = exports.secret = exports.removeSecretsFromEnv = exports.loadSecretsFromJsonFile = exports.loadSecretsFromEnv = exports.setSecretMap = exports.getSecretMap = exports.Debug = exports.base64ToBuffer = exports.bufferToBase64 = exports.base64ToString = exports.stringToBase64 = exports.md5AsBuffer = exports.hashAsBuffer = exports.hash = exports.md5 = exports.ALPHABET_ALPHANUMERIC = exports.ALPHABET_ALPHANUMERIC_UPPERCASE = exports.ALPHABET_ALPHANUMERIC_LOWERCASE = exports.ALPHABET_UPPERCASE = void 0;
5
- exports.sanitizeHTML = exports.hasColors = exports.readAjvSchemas = exports.readJsonSchemas = exports.AjvValidationError = exports.AjvSchema = exports.getAjv = exports.Ajv = exports._chunkBuffer = exports.TimeoutError = exports.HTTPError = exports.getGot = exports.inspectAny = exports.tableDiff = exports.BaseWorkerClass = exports.transformMultiThreaded = exports.writableFork = exports.writableVoid = exports.transformLimit = exports.transformLogProgress = exports.transformTap = exports.transformToArray = exports.transformToString = exports.transformSplit = exports.writablePushToArray = exports.writableForEach = exports.transformNoOp = void 0;
5
+ exports.runScript = exports.sanitizeHTML = exports.hasColors = exports.readAjvSchemas = exports.readJsonSchemas = exports.AjvValidationError = exports.AjvSchema = exports.getAjv = exports.Ajv = exports._chunkBuffer = exports.TimeoutError = exports.HTTPError = exports.getGot = exports.inspectAnyStringifyFn = exports.inspectAny = exports.tableDiff = exports.BaseWorkerClass = exports.transformMultiThreaded = exports.writableFork = exports.writableVoid = exports.transformLimit = exports.transformLogProgress = exports.transformTap = exports.transformToArray = exports.transformToString = exports.transformSplit = exports.writablePushToArray = exports.writableForEach = exports.transformNoOp = void 0;
6
6
  const ajv_1 = require("ajv");
7
7
  exports.Ajv = ajv_1.default;
8
8
  const got_1 = require("got");
@@ -128,6 +128,7 @@ const writableVoid_1 = require("./stream/writable/writableVoid");
128
128
  Object.defineProperty(exports, "writableVoid", { enumerable: true, get: function () { return writableVoid_1.writableVoid; } });
129
129
  const inspectAny_1 = require("./string/inspectAny");
130
130
  Object.defineProperty(exports, "inspectAny", { enumerable: true, get: function () { return inspectAny_1.inspectAny; } });
131
+ Object.defineProperty(exports, "inspectAnyStringifyFn", { enumerable: true, get: function () { return inspectAny_1.inspectAnyStringifyFn; } });
131
132
  const env_util_1 = require("./util/env.util");
132
133
  Object.defineProperty(exports, "requireEnvKeys", { enumerable: true, get: function () { return env_util_1.requireEnvKeys; } });
133
134
  Object.defineProperty(exports, "requireFileToExist", { enumerable: true, get: function () { return env_util_1.requireFileToExist; } });
@@ -190,3 +191,5 @@ Object.defineProperty(exports, "undefinedIfInvalid", { enumerable: true, get: fu
190
191
  Object.defineProperty(exports, "validate", { enumerable: true, get: function () { return joi_validation_util_1.validate; } });
191
192
  const sanitize_util_1 = require("./validation/sanitize.util");
192
193
  Object.defineProperty(exports, "sanitizeHTML", { enumerable: true, get: function () { return sanitize_util_1.sanitizeHTML; } });
194
+ const script_1 = require("./script");
195
+ Object.defineProperty(exports, "runScript", { enumerable: true, get: function () { return script_1.runScript; } });
@@ -1,3 +1,4 @@
1
+ import type { CommonLogger } from '@naturalcycles/js-lib';
1
2
  export interface RunScriptOptions {
2
3
  /**
3
4
  * @default false
@@ -5,6 +6,10 @@ export interface RunScriptOptions {
5
6
  * Currently it exists because of `jest --maxWorkers=1` behavior. To be investigated more..
6
7
  */
7
8
  noExit?: boolean;
9
+ /**
10
+ * Default to `console`
11
+ */
12
+ logger?: CommonLogger;
8
13
  }
9
14
  /**
10
15
  * Use it in your top-level scripts like this:
@@ -18,23 +18,24 @@ exports.runScript = void 0;
18
18
  * This function is kept light, dependency-free, exported separately.
19
19
  */
20
20
  function runScript(fn, opt = {}) {
21
+ const { logger = console, noExit } = opt;
21
22
  process.on('uncaughtException', err => {
22
- console.error('uncaughtException', err);
23
+ logger.error('uncaughtException:', err);
23
24
  });
24
25
  process.on('unhandledRejection', err => {
25
- console.error('unhandledRejection', err);
26
+ logger.error('unhandledRejection:', err);
26
27
  });
27
28
  void (async () => {
28
29
  try {
29
30
  await fn();
30
- if (!opt.noExit) {
31
+ if (!noExit) {
31
32
  setImmediate(() => process.exit(0));
32
33
  }
33
34
  }
34
35
  catch (err) {
35
- console.error('runScript failed:', err);
36
+ logger.error('runScript error:', err);
36
37
  process.exitCode = 1;
37
- if (!opt.noExit) {
38
+ if (!noExit) {
38
39
  setImmediate(() => process.exit(1));
39
40
  }
40
41
  }
@@ -1,4 +1,4 @@
1
- import { StringMap } from '@naturalcycles/js-lib';
1
+ import { AnyObject, CommonLogger, CommonLogLevel } from '@naturalcycles/js-lib';
2
2
  import { SlackAttachmentField, SlackMessage, SlackServiceCfg } from './slack.service.model';
3
3
  /**
4
4
  * Has 2 main methods:
@@ -20,6 +20,15 @@ export declare class SlackService<CTX = any> {
20
20
  */
21
21
  log(...items: any[]): Promise<void>;
22
22
  send(input: SlackMessage<CTX> | string, ctx?: CTX): Promise<void>;
23
- kvToFields(kv: StringMap<any>): SlackAttachmentField[];
23
+ kvToFields(kv: AnyObject): SlackAttachmentField[];
24
+ /**
25
+ * Returns a CommonLogger implementation based on this SlackService instance.
26
+ */
27
+ getCommonLogger(opt: {
28
+ minLogLevel: CommonLogLevel;
29
+ logChannel?: string;
30
+ warnChannel?: string;
31
+ errorChannel?: string;
32
+ }): CommonLogger;
24
33
  }
25
34
  export declare function slackDefaultMessagePrefixHook(msg: SlackMessage): string[];
@@ -110,6 +110,21 @@ class SlackService {
110
110
  short: String(v).length < 80,
111
111
  }));
112
112
  }
113
+ /**
114
+ * Returns a CommonLogger implementation based on this SlackService instance.
115
+ */
116
+ getCommonLogger(opt) {
117
+ const { minLogLevel = 'log', logChannel, warnChannel, errorChannel } = opt;
118
+ const defaultChannel = this.cfg.defaults?.channel || DEFAULTS.channel;
119
+ const q = new js_lib_1.PQueue({
120
+ concurrency: 1,
121
+ });
122
+ return (0, js_lib_1.commonLoggerMinLevel)({
123
+ log: (...args) => q.push(() => this.send({ items: args, channel: logChannel || defaultChannel })),
124
+ warn: (...args) => q.push(() => this.send({ items: args, channel: warnChannel || defaultChannel })),
125
+ error: (...args) => q.push(() => this.send({ items: args, channel: errorChannel || defaultChannel })),
126
+ }, minLogLevel);
127
+ }
113
128
  }
114
129
  exports.SlackService = SlackService;
115
130
  function slackDefaultMessagePrefixHook(msg) {
@@ -1,4 +1,4 @@
1
- import { CommonLogger, StringMap } from '@naturalcycles/js-lib';
1
+ import { AnyObject, CommonLogger } from '@naturalcycles/js-lib';
2
2
  import { InspectAnyOptions } from '..';
3
3
  /**
4
4
  * Properties that exists both in SlackApiBody (as per Slack API) and SlackMessage (our abstraction).
@@ -39,7 +39,7 @@ export interface SlackMessage<CTX = any> extends SlackMessageProps {
39
39
  /**
40
40
  * Keys-values will be rendered as MessageAttachment with Fields
41
41
  */
42
- kv?: StringMap<any>;
42
+ kv?: AnyObject;
43
43
  /**
44
44
  * If specified - adds @name1, @name2 in the end of the message
45
45
  */
@@ -16,6 +16,10 @@ export interface TransformToNDJsonOptions {
16
16
  * @default `\n`
17
17
  */
18
18
  separator?: string;
19
+ /**
20
+ * @experimental
21
+ */
22
+ useFlatstr?: boolean;
19
23
  }
20
24
  /**
21
25
  * Transforms objects (objectMode=true) into chunks \n-terminated JSON strings (readableObjectMode=false).
@@ -7,7 +7,7 @@ const js_lib_1 = require("@naturalcycles/js-lib");
7
7
  * Transforms objects (objectMode=true) into chunks \n-terminated JSON strings (readableObjectMode=false).
8
8
  */
9
9
  function transformToNDJson(opt = {}) {
10
- const { strict = true, separator = '\n', sortObjects = false } = opt;
10
+ const { strict = true, separator = '\n', sortObjects = false, useFlatstr = false } = opt;
11
11
  return new stream_1.Transform({
12
12
  objectMode: true,
13
13
  readableObjectMode: false,
@@ -16,7 +16,12 @@ function transformToNDJson(opt = {}) {
16
16
  if (sortObjects) {
17
17
  chunk = (0, js_lib_1._sortObjectDeep)(chunk);
18
18
  }
19
- cb(null, JSON.stringify(chunk) + separator);
19
+ if (useFlatstr) {
20
+ cb(null, flatstr(JSON.stringify(chunk) + separator));
21
+ }
22
+ else {
23
+ cb(null, JSON.stringify(chunk) + separator);
24
+ }
20
25
  }
21
26
  catch (err) {
22
27
  console.error(err);
@@ -31,3 +36,11 @@ function transformToNDJson(opt = {}) {
31
36
  });
32
37
  }
33
38
  exports.transformToNDJson = transformToNDJson;
39
+ /**
40
+ * Based on: https://github.com/davidmarkclements/flatstr/blob/master/index.js
41
+ */
42
+ function flatstr(s) {
43
+ // eslint-disable-next-line
44
+ s | 0;
45
+ return s;
46
+ }
@@ -0,0 +1,17 @@
1
+ import { AsyncMapper } from '@naturalcycles/js-lib';
2
+ import { TransformTyped } from '../../stream.model';
3
+ import { TransformMapOptions } from '../transformMap';
4
+ export declare function notNullishPredicate(item: any): boolean;
5
+ /**
6
+ * Like pMap, but for streams.
7
+ * Inspired by `through2`.
8
+ * Main feature is concurrency control (implemented via `through2-concurrent`) and convenient options.
9
+ * Using this allows native stream .pipe() to work and use backpressure.
10
+ *
11
+ * Only works in objectMode (due to through2Concurrent).
12
+ *
13
+ * Concurrency defaults to 16.
14
+ *
15
+ * If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
16
+ */
17
+ export declare function transformMap2<IN = any, OUT = IN>(mapper: AsyncMapper<IN, OUT>, opt?: TransformMapOptions<IN, OUT>): TransformTyped<IN, OUT>;
@@ -0,0 +1,98 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.transformMap2 = exports.notNullishPredicate = void 0;
4
+ const stream_1 = require("stream");
5
+ const js_lib_1 = require("@naturalcycles/js-lib");
6
+ const colors_1 = require("../../../colors");
7
+ function notNullishPredicate(item) {
8
+ return item !== undefined && item !== null;
9
+ }
10
+ exports.notNullishPredicate = notNullishPredicate;
11
+ /**
12
+ * Like pMap, but for streams.
13
+ * Inspired by `through2`.
14
+ * Main feature is concurrency control (implemented via `through2-concurrent`) and convenient options.
15
+ * Using this allows native stream .pipe() to work and use backpressure.
16
+ *
17
+ * Only works in objectMode (due to through2Concurrent).
18
+ *
19
+ * Concurrency defaults to 16.
20
+ *
21
+ * If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
22
+ */
23
+ function transformMap2(mapper, opt = {}) {
24
+ const { concurrency = 16, predicate = notNullishPredicate, errorMode = js_lib_1.ErrorMode.THROW_IMMEDIATELY, flattenArrayOutput, onError, beforeFinal, metric = 'stream', logger = console, } = opt;
25
+ let index = -1;
26
+ let isRejected = false;
27
+ let errors = 0;
28
+ const collectedErrors = []; // only used if errorMode == THROW_AGGREGATED
29
+ const q = new js_lib_1.PQueue({
30
+ concurrency,
31
+ resolveOn: 'start',
32
+ // debug: true,
33
+ });
34
+ return new stream_1.Transform({
35
+ objectMode: true,
36
+ async final(cb) {
37
+ // console.log('transformMap final', {index}, q.inFlight, q.queueSize)
38
+ // wait for the current inFlight jobs to complete and push their results
39
+ await q.onIdle();
40
+ logErrorStats(logger, true);
41
+ await beforeFinal?.(); // call beforeFinal if defined
42
+ if (collectedErrors.length) {
43
+ // emit Aggregated error
44
+ cb(new js_lib_1.AggregatedError(collectedErrors));
45
+ }
46
+ else {
47
+ // emit no error
48
+ cb();
49
+ }
50
+ },
51
+ async transform(chunk, _encoding, cb) {
52
+ index++;
53
+ // console.log('transform', {index})
54
+ // Stop processing if THROW_IMMEDIATELY mode is used
55
+ if (isRejected && errorMode === js_lib_1.ErrorMode.THROW_IMMEDIATELY)
56
+ return cb();
57
+ // It resolves when it is successfully STARTED execution.
58
+ // If it's queued instead - it'll wait and resolve only upon START.
59
+ await q.push(async () => {
60
+ try {
61
+ const currentIndex = index; // because we need to pass it to 2 functions - mapper and predicate. Refers to INPUT index (since it may return multiple outputs)
62
+ const res = await mapper(chunk, currentIndex);
63
+ const passedResults = await (0, js_lib_1.pFilter)(flattenArrayOutput && Array.isArray(res) ? res : [res], async (r) => await predicate(r, currentIndex));
64
+ passedResults.forEach(r => this.push(r));
65
+ }
66
+ catch (err) {
67
+ logger.error(err);
68
+ errors++;
69
+ logErrorStats(logger);
70
+ if (onError) {
71
+ try {
72
+ onError(err, chunk);
73
+ }
74
+ catch { }
75
+ }
76
+ if (errorMode === js_lib_1.ErrorMode.THROW_IMMEDIATELY) {
77
+ isRejected = true;
78
+ // Emit error immediately
79
+ // return cb(err as Error)
80
+ return this.emit('error', err);
81
+ }
82
+ if (errorMode === js_lib_1.ErrorMode.THROW_AGGREGATED) {
83
+ collectedErrors.push(err);
84
+ }
85
+ }
86
+ });
87
+ // Resolved, which means it STARTED processing
88
+ // This means we can take more load
89
+ cb();
90
+ },
91
+ });
92
+ function logErrorStats(logger, final = false) {
93
+ if (!errors)
94
+ return;
95
+ logger.log(`${metric} ${final ? 'final ' : ''}errors: ${(0, colors_1.yellow)(errors)}`);
96
+ }
97
+ }
98
+ exports.transformMap2 = transformMap2;
@@ -1,4 +1,4 @@
1
- import { AsyncMapper, AsyncPredicate, ErrorMode } from '@naturalcycles/js-lib';
1
+ import { AsyncMapper, AsyncPredicate, CommonLogger, ErrorMode } from '@naturalcycles/js-lib';
2
2
  import { TransformTyped } from '../stream.model';
3
3
  export interface TransformMapOptions<IN = any, OUT = IN> {
4
4
  /**
@@ -41,6 +41,7 @@ export interface TransformMapOptions<IN = any, OUT = IN> {
41
41
  * If defined - called BEFORE `final()` callback is called.
42
42
  */
43
43
  beforeFinal?: () => any;
44
+ logger?: CommonLogger;
44
45
  }
45
46
  export declare function notNullishPredicate(item: any): boolean;
46
47
  /**
@@ -21,7 +21,7 @@ exports.notNullishPredicate = notNullishPredicate;
21
21
  * If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
22
22
  */
23
23
  function transformMap(mapper, opt = {}) {
24
- const { concurrency = 16, predicate = notNullishPredicate, errorMode = js_lib_1.ErrorMode.THROW_IMMEDIATELY, flattenArrayOutput, onError, beforeFinal, metric = 'stream', } = opt;
24
+ const { concurrency = 16, predicate = notNullishPredicate, errorMode = js_lib_1.ErrorMode.THROW_IMMEDIATELY, flattenArrayOutput, onError, beforeFinal, metric = 'stream', logger = console, } = opt;
25
25
  let index = -1;
26
26
  let isRejected = false;
27
27
  let errors = 0;
@@ -31,7 +31,7 @@ function transformMap(mapper, opt = {}) {
31
31
  // autoDestroy: true,
32
32
  async final(cb) {
33
33
  // console.log('transformMap final')
34
- logErrorStats(true);
34
+ logErrorStats(logger, true);
35
35
  await beforeFinal?.(); // call beforeFinal if defined
36
36
  if (collectedErrors.length) {
37
37
  // emit Aggregated error
@@ -64,9 +64,9 @@ function transformMap(mapper, opt = {}) {
64
64
  }
65
65
  }
66
66
  catch (err) {
67
- console.error(err);
67
+ logger.error(err);
68
68
  errors++;
69
- logErrorStats();
69
+ logErrorStats(logger);
70
70
  if (onError) {
71
71
  try {
72
72
  onError(err, chunk);
@@ -85,10 +85,10 @@ function transformMap(mapper, opt = {}) {
85
85
  cb();
86
86
  }
87
87
  });
88
- function logErrorStats(final = false) {
88
+ function logErrorStats(logger, final = false) {
89
89
  if (!errors)
90
90
  return;
91
- console.log(`${metric} ${final ? 'final ' : ''}errors: ${(0, colors_1.yellow)(errors)}`);
91
+ logger.log(`${metric} ${final ? 'final ' : ''}errors: ${(0, colors_1.yellow)(errors)}`);
92
92
  }
93
93
  }
94
94
  exports.transformMap = transformMap;
@@ -1,8 +1,12 @@
1
1
  /// <reference types="node" />
2
2
  import { InspectOptions } from 'util';
3
- import { StringifyAnyOptions } from '@naturalcycles/js-lib';
3
+ import { StringifyAnyOptions, JsonStringifyFunction } from '@naturalcycles/js-lib';
4
4
  export interface InspectAnyOptions extends StringifyAnyOptions, InspectOptions {
5
5
  }
6
+ /**
7
+ * Just a convenience export of a const that fulfills the JsonStringifyFunction interface.
8
+ */
9
+ export declare const inspectAnyStringifyFn: JsonStringifyFunction;
6
10
  /**
7
11
  * Transforms ANY to human-readable string (via util.inspect mainly).
8
12
  * Safe (no error throwing).
@@ -1,12 +1,17 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.inspectAny = void 0;
3
+ exports.inspectAny = exports.inspectAnyStringifyFn = void 0;
4
4
  const util_1 = require("util");
5
5
  const js_lib_1 = require("@naturalcycles/js-lib");
6
6
  const INSPECT_OPT = {
7
7
  breakLength: 80,
8
8
  depth: 10, // default: 2
9
9
  };
10
+ /**
11
+ * Just a convenience export of a const that fulfills the JsonStringifyFunction interface.
12
+ */
13
+ const inspectAnyStringifyFn = obj => inspectAny(obj);
14
+ exports.inspectAnyStringifyFn = inspectAnyStringifyFn;
10
15
  /**
11
16
  * Transforms ANY to human-readable string (via util.inspect mainly).
12
17
  * Safe (no error throwing).
@@ -1,4 +1,4 @@
1
- import { JsonSchema, JsonSchemaBuilder } from '@naturalcycles/js-lib';
1
+ import { JsonSchema, JsonSchemaBuilder, CommonLogger } from '@naturalcycles/js-lib';
2
2
  import Ajv from 'ajv';
3
3
  import { AjvValidationError } from './ajvValidationError';
4
4
  export interface AjvValidationOptions {
@@ -37,6 +37,10 @@ export interface AjvSchemaCfg {
37
37
  * @default true
38
38
  */
39
39
  logErrors: boolean;
40
+ /**
41
+ * Default to `console`
42
+ */
43
+ logger: CommonLogger;
40
44
  /**
41
45
  * Option of Ajv.
42
46
  * If set to true - will mutate your input objects!
@@ -17,6 +17,7 @@ class AjvSchema {
17
17
  this.schema = schema;
18
18
  this.cfg = {
19
19
  logErrors: true,
20
+ logger: console,
20
21
  separator: '\n',
21
22
  ...cfg,
22
23
  ajv: cfg.ajv ||
@@ -89,7 +90,7 @@ class AjvSchema {
89
90
  const strValue = (0, index_1.inspectAny)(obj, { maxLen: 1000 });
90
91
  message = [message, 'Input: ' + strValue].join(separator);
91
92
  if (logErrors) {
92
- console.log(errors);
93
+ this.cfg.logger.error(errors);
93
94
  }
94
95
  return new ajvValidationError_1.AjvValidationError(message, (0, js_lib_1._filterNullishValues)({
95
96
  errors,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@naturalcycles/nodejs-lib",
3
- "version": "12.52.0",
3
+ "version": "12.55.1",
4
4
  "scripts": {
5
5
  "prepare": "husky install",
6
6
  "docs-serve": "vuepress dev docs",
@@ -1,4 +1,4 @@
1
- import { StringMap, _truncate, AnyObject } from '@naturalcycles/js-lib'
1
+ import { _truncate, AnyObject } from '@naturalcycles/js-lib'
2
2
 
3
3
  export interface TableDiffOptions {
4
4
  /**
@@ -37,7 +37,7 @@ export interface TableDiffOptions {
37
37
  */
38
38
  export function tableDiff(a: AnyObject, b: AnyObject, opt: TableDiffOptions = {}): void {
39
39
  const { maxFieldLen, aTitle = 'a', bTitle = 'b' } = opt
40
- const diff: StringMap<any> = {}
40
+ const diff: AnyObject = {}
41
41
 
42
42
  if (a && b && a !== b) {
43
43
  new Set([...Object.keys(a), ...Object.keys(b)]).forEach(k => {
package/src/got/getGot.ts CHANGED
@@ -1,5 +1,5 @@
1
1
  import { URL } from 'url'
2
- import { _jsonParseIfPossible, _since } from '@naturalcycles/js-lib'
2
+ import { _since } from '@naturalcycles/js-lib'
3
3
  import got, { AfterResponseHook, BeforeErrorHook, BeforeRequestHook, Got, HTTPError } from 'got'
4
4
  import { inspectAny } from '..'
5
5
  import { dimGrey, grey, red, yellow } from '../colors'
@@ -13,6 +13,8 @@ import { GetGotOptions, GotRequestContext } from './got.model'
13
13
  * 3. Reasonable defaults(tm), e.g non-infinite Timeout
14
14
  */
15
15
  export function getGot(opt: GetGotOptions = {}): Got {
16
+ opt.logger ||= console
17
+
16
18
  return got.extend({
17
19
  // Most-important is to set to anything non-empty (so, requests don't "hang" by default).
18
20
  // Should be long enough to handle for slow responses from scaled cloud APIs in times of spikes
@@ -99,7 +101,7 @@ function gotBeforeRequestHook(opt: GetGotOptions): BeforeRequestHook {
99
101
 
100
102
  if (opt.logStart) {
101
103
  const shortUrl = getShortUrl(opt, options.url, options.prefixUrl)
102
- console.log([dimGrey(' >>'), dimGrey(options.method), grey(shortUrl)].join(' '))
104
+ opt.logger!.log([dimGrey(' >>'), dimGrey(options.method), grey(shortUrl)].join(' '))
103
105
  }
104
106
  }
105
107
  }
@@ -113,7 +115,7 @@ function gotAfterResponseHook(opt: GetGotOptions = {}): AfterResponseHook {
113
115
  const { url, prefixUrl, method } = resp.request.options
114
116
  const shortUrl = getShortUrl(opt, url, prefixUrl)
115
117
 
116
- console.log(
118
+ opt.logger!.log(
117
119
  [
118
120
  dimGrey(' <<'),
119
121
  coloredHttpCode(resp.statusCode),
@@ -129,7 +131,7 @@ function gotAfterResponseHook(opt: GetGotOptions = {}): AfterResponseHook {
129
131
 
130
132
  // Error responses are not logged, cause they're included in Error message already
131
133
  if (opt.logResponse && success) {
132
- console.log(inspectAny(_jsonParseIfPossible(resp.body), { maxLen: opt.maxResponseLength }))
134
+ opt.logger!.log(inspectAny(resp.body, { maxLen: opt.maxResponseLength }))
133
135
  }
134
136
 
135
137
  return resp
@@ -1,4 +1,4 @@
1
- import { AnyObject } from '@naturalcycles/js-lib'
1
+ import { AnyObject, CommonLogger } from '@naturalcycles/js-lib'
2
2
  import type { Options } from 'got'
3
3
 
4
4
  export interface GetGotOptions extends Options {
@@ -33,6 +33,11 @@ export interface GetGotOptions extends Options {
33
33
  */
34
34
  logWithSearchParams?: boolean
35
35
 
36
+ /**
37
+ * Defaults to `console`
38
+ */
39
+ logger?: CommonLogger
40
+
36
41
  /**
37
42
  * Max length of response object before it's truncated.
38
43
  *
package/src/index.ts CHANGED
@@ -108,7 +108,7 @@ import { writableForEach } from './stream/writable/writableForEach'
108
108
  import { writableFork } from './stream/writable/writableFork'
109
109
  import { writablePushToArray } from './stream/writable/writablePushToArray'
110
110
  import { writableVoid } from './stream/writable/writableVoid'
111
- import { inspectAny, InspectAnyOptions } from './string/inspectAny'
111
+ import { inspectAny, InspectAnyOptions, inspectAnyStringifyFn } from './string/inspectAny'
112
112
  import { requireEnvKeys, requireFileToExist } from './util/env.util'
113
113
  import { LRUMemoCache } from './util/lruMemoCache'
114
114
  import {
@@ -173,8 +173,10 @@ import {
173
173
  validate,
174
174
  } from './validation/joi/joi.validation.util'
175
175
  import { sanitizeHTML, SanitizeHTMLOptions } from './validation/sanitize.util'
176
+ import { runScript, RunScriptOptions } from './script'
176
177
 
177
178
  export type {
179
+ RunScriptOptions,
178
180
  JoiValidationErrorData,
179
181
  JoiValidationResult,
180
182
  ValidationErrorItem,
@@ -339,6 +341,7 @@ export {
339
341
  BaseWorkerClass,
340
342
  tableDiff,
341
343
  inspectAny,
344
+ inspectAnyStringifyFn,
342
345
  getGot,
343
346
  HTTPError,
344
347
  TimeoutError,
@@ -351,4 +354,5 @@ export {
351
354
  readAjvSchemas,
352
355
  hasColors,
353
356
  sanitizeHTML,
357
+ runScript,
354
358
  }
@@ -1,3 +1,5 @@
1
+ import type { CommonLogger } from '@naturalcycles/js-lib'
2
+
1
3
  export interface RunScriptOptions {
2
4
  /**
3
5
  * @default false
@@ -5,6 +7,11 @@ export interface RunScriptOptions {
5
7
  * Currently it exists because of `jest --maxWorkers=1` behavior. To be investigated more..
6
8
  */
7
9
  noExit?: boolean
10
+
11
+ /**
12
+ * Default to `console`
13
+ */
14
+ logger?: CommonLogger
8
15
  }
9
16
 
10
17
  /**
@@ -24,24 +31,26 @@ export interface RunScriptOptions {
24
31
  * This function is kept light, dependency-free, exported separately.
25
32
  */
26
33
  export function runScript(fn: (...args: any[]) => any, opt: RunScriptOptions = {}): void {
34
+ const { logger = console, noExit } = opt
35
+
27
36
  process.on('uncaughtException', err => {
28
- console.error('uncaughtException', err)
37
+ logger.error('uncaughtException:', err)
29
38
  })
30
39
  process.on('unhandledRejection', err => {
31
- console.error('unhandledRejection', err)
40
+ logger.error('unhandledRejection:', err)
32
41
  })
33
42
 
34
43
  void (async () => {
35
44
  try {
36
45
  await fn()
37
46
 
38
- if (!opt.noExit) {
47
+ if (!noExit) {
39
48
  setImmediate(() => process.exit(0))
40
49
  }
41
50
  } catch (err) {
42
- console.error('runScript failed:', err)
51
+ logger.error('runScript error:', err)
43
52
  process.exitCode = 1
44
- if (!opt.noExit) {
53
+ if (!noExit) {
45
54
  setImmediate(() => process.exit(1))
46
55
  }
47
56
  }
@@ -1,4 +1,4 @@
1
- import { CommonLogger, StringMap } from '@naturalcycles/js-lib'
1
+ import { AnyObject, CommonLogger } from '@naturalcycles/js-lib'
2
2
  import { InspectAnyOptions } from '..'
3
3
 
4
4
  /**
@@ -47,7 +47,7 @@ export interface SlackMessage<CTX = any> extends SlackMessageProps {
47
47
  /**
48
48
  * Keys-values will be rendered as MessageAttachment with Fields
49
49
  */
50
- kv?: StringMap<any>
50
+ kv?: AnyObject
51
51
 
52
52
  /**
53
53
  * If specified - adds @name1, @name2 in the end of the message
@@ -1,4 +1,11 @@
1
- import { _omit, StringMap } from '@naturalcycles/js-lib'
1
+ import {
2
+ _omit,
3
+ AnyObject,
4
+ CommonLogger,
5
+ commonLoggerMinLevel,
6
+ CommonLogLevel,
7
+ PQueue,
8
+ } from '@naturalcycles/js-lib'
2
9
  import { dayjs } from '@naturalcycles/time-lib'
3
10
  import got from 'got'
4
11
  import { inspectAny, InspectAnyOptions } from '..'
@@ -126,13 +133,42 @@ export class SlackService<CTX = any> {
126
133
  })
127
134
  }
128
135
 
129
- kvToFields(kv: StringMap<any>): SlackAttachmentField[] {
136
+ kvToFields(kv: AnyObject): SlackAttachmentField[] {
130
137
  return Object.entries(kv).map(([k, v]) => ({
131
138
  title: k,
132
139
  value: String(v),
133
140
  short: String(v).length < 80,
134
141
  }))
135
142
  }
143
+
144
+ /**
145
+ * Returns a CommonLogger implementation based on this SlackService instance.
146
+ */
147
+ getCommonLogger(opt: {
148
+ minLogLevel: CommonLogLevel
149
+ logChannel?: string
150
+ warnChannel?: string
151
+ errorChannel?: string
152
+ }): CommonLogger {
153
+ const { minLogLevel = 'log', logChannel, warnChannel, errorChannel } = opt
154
+ const defaultChannel = this.cfg.defaults?.channel || DEFAULTS.channel!
155
+
156
+ const q = new PQueue({
157
+ concurrency: 1,
158
+ })
159
+
160
+ return commonLoggerMinLevel(
161
+ {
162
+ log: (...args) =>
163
+ q.push(() => this.send({ items: args, channel: logChannel || defaultChannel })),
164
+ warn: (...args) =>
165
+ q.push(() => this.send({ items: args, channel: warnChannel || defaultChannel })),
166
+ error: (...args) =>
167
+ q.push(() => this.send({ items: args, channel: errorChannel || defaultChannel })),
168
+ },
169
+ minLogLevel,
170
+ )
171
+ }
136
172
  }
137
173
 
138
174
  export function slackDefaultMessagePrefixHook(msg: SlackMessage): string[] {
@@ -21,6 +21,11 @@ export interface TransformToNDJsonOptions {
21
21
  * @default `\n`
22
22
  */
23
23
  separator?: string
24
+
25
+ /**
26
+ * @experimental
27
+ */
28
+ useFlatstr?: boolean
24
29
  }
25
30
 
26
31
  /**
@@ -29,7 +34,7 @@ export interface TransformToNDJsonOptions {
29
34
  export function transformToNDJson<IN = any>(
30
35
  opt: TransformToNDJsonOptions = {},
31
36
  ): TransformTyped<IN, string> {
32
- const { strict = true, separator = '\n', sortObjects = false } = opt
37
+ const { strict = true, separator = '\n', sortObjects = false, useFlatstr = false } = opt
33
38
 
34
39
  return new Transform({
35
40
  objectMode: true,
@@ -39,7 +44,12 @@ export function transformToNDJson<IN = any>(
39
44
  if (sortObjects) {
40
45
  chunk = _sortObjectDeep(chunk as any)
41
46
  }
42
- cb(null, JSON.stringify(chunk) + separator)
47
+
48
+ if (useFlatstr) {
49
+ cb(null, flatstr(JSON.stringify(chunk) + separator))
50
+ } else {
51
+ cb(null, JSON.stringify(chunk) + separator)
52
+ }
43
53
  } catch (err) {
44
54
  console.error(err)
45
55
 
@@ -52,3 +62,12 @@ export function transformToNDJson<IN = any>(
52
62
  },
53
63
  })
54
64
  }
65
+
66
+ /**
67
+ * Based on: https://github.com/davidmarkclements/flatstr/blob/master/index.js
68
+ */
69
+ function flatstr(s: any): string {
70
+ // eslint-disable-next-line
71
+ s | 0
72
+ return s
73
+ }
@@ -0,0 +1,134 @@
1
+ import { Transform } from 'stream'
2
+ import {
3
+ AggregatedError,
4
+ AsyncMapper,
5
+ CommonLogger,
6
+ ErrorMode,
7
+ pFilter,
8
+ PQueue,
9
+ } from '@naturalcycles/js-lib'
10
+ import { yellow } from '../../../colors'
11
+ import { TransformTyped } from '../../stream.model'
12
+ import { TransformMapOptions } from '../transformMap'
13
+
14
+ export function notNullishPredicate(item: any): boolean {
15
+ return item !== undefined && item !== null
16
+ }
17
+
18
+ /**
19
+ * Like pMap, but for streams.
20
+ * Inspired by `through2`.
21
+ * Main feature is concurrency control (implemented via `through2-concurrent`) and convenient options.
22
+ * Using this allows native stream .pipe() to work and use backpressure.
23
+ *
24
+ * Only works in objectMode (due to through2Concurrent).
25
+ *
26
+ * Concurrency defaults to 16.
27
+ *
28
+ * If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
29
+ */
30
+ export function transformMap2<IN = any, OUT = IN>(
31
+ mapper: AsyncMapper<IN, OUT>,
32
+ opt: TransformMapOptions<IN, OUT> = {},
33
+ ): TransformTyped<IN, OUT> {
34
+ const {
35
+ concurrency = 16,
36
+ predicate = notNullishPredicate,
37
+ errorMode = ErrorMode.THROW_IMMEDIATELY,
38
+ flattenArrayOutput,
39
+ onError,
40
+ beforeFinal,
41
+ metric = 'stream',
42
+ logger = console,
43
+ } = opt
44
+
45
+ let index = -1
46
+ let isRejected = false
47
+ let errors = 0
48
+ const collectedErrors: Error[] = [] // only used if errorMode == THROW_AGGREGATED
49
+
50
+ const q = new PQueue({
51
+ concurrency,
52
+ resolveOn: 'start',
53
+ // debug: true,
54
+ })
55
+
56
+ return new Transform({
57
+ objectMode: true,
58
+
59
+ async final(cb) {
60
+ // console.log('transformMap final', {index}, q.inFlight, q.queueSize)
61
+
62
+ // wait for the current inFlight jobs to complete and push their results
63
+ await q.onIdle()
64
+
65
+ logErrorStats(logger, true)
66
+
67
+ await beforeFinal?.() // call beforeFinal if defined
68
+
69
+ if (collectedErrors.length) {
70
+ // emit Aggregated error
71
+ cb(new AggregatedError(collectedErrors))
72
+ } else {
73
+ // emit no error
74
+ cb()
75
+ }
76
+ },
77
+
78
+ async transform(this: Transform, chunk: IN, _encoding, cb) {
79
+ index++
80
+ // console.log('transform', {index})
81
+
82
+ // Stop processing if THROW_IMMEDIATELY mode is used
83
+ if (isRejected && errorMode === ErrorMode.THROW_IMMEDIATELY) return cb()
84
+
85
+ // It resolves when it is successfully STARTED execution.
86
+ // If it's queued instead - it'll wait and resolve only upon START.
87
+ await q.push(async () => {
88
+ try {
89
+ const currentIndex = index // because we need to pass it to 2 functions - mapper and predicate. Refers to INPUT index (since it may return multiple outputs)
90
+ const res = await mapper(chunk, currentIndex)
91
+ const passedResults = await pFilter(
92
+ flattenArrayOutput && Array.isArray(res) ? res : [res],
93
+ async r => await predicate(r, currentIndex),
94
+ )
95
+
96
+ passedResults.forEach(r => this.push(r))
97
+ } catch (err) {
98
+ logger.error(err)
99
+
100
+ errors++
101
+
102
+ logErrorStats(logger)
103
+
104
+ if (onError) {
105
+ try {
106
+ onError(err, chunk)
107
+ } catch {}
108
+ }
109
+
110
+ if (errorMode === ErrorMode.THROW_IMMEDIATELY) {
111
+ isRejected = true
112
+ // Emit error immediately
113
+ // return cb(err as Error)
114
+ return this.emit('error', err as Error)
115
+ }
116
+
117
+ if (errorMode === ErrorMode.THROW_AGGREGATED) {
118
+ collectedErrors.push(err as Error)
119
+ }
120
+ }
121
+ })
122
+
123
+ // Resolved, which means it STARTED processing
124
+ // This means we can take more load
125
+ cb()
126
+ },
127
+ })
128
+
129
+ function logErrorStats(logger: CommonLogger, final = false): void {
130
+ if (!errors) return
131
+
132
+ logger.log(`${metric} ${final ? 'final ' : ''}errors: ${yellow(errors)}`)
133
+ }
134
+ }
@@ -3,6 +3,7 @@ import {
3
3
  AggregatedError,
4
4
  AsyncMapper,
5
5
  AsyncPredicate,
6
+ CommonLogger,
6
7
  ErrorMode,
7
8
  pFilter,
8
9
  } from '@naturalcycles/js-lib'
@@ -57,6 +58,8 @@ export interface TransformMapOptions<IN = any, OUT = IN> {
57
58
  * If defined - called BEFORE `final()` callback is called.
58
59
  */
59
60
  beforeFinal?: () => any
61
+
62
+ logger?: CommonLogger
60
63
  }
61
64
 
62
65
  export function notNullishPredicate(item: any): boolean {
@@ -87,6 +90,7 @@ export function transformMap<IN = any, OUT = IN>(
87
90
  onError,
88
91
  beforeFinal,
89
92
  metric = 'stream',
93
+ logger = console,
90
94
  } = opt
91
95
 
92
96
  let index = -1
@@ -101,7 +105,7 @@ export function transformMap<IN = any, OUT = IN>(
101
105
  async final(cb) {
102
106
  // console.log('transformMap final')
103
107
 
104
- logErrorStats(true)
108
+ logErrorStats(logger, true)
105
109
 
106
110
  await beforeFinal?.() // call beforeFinal if defined
107
111
 
@@ -144,11 +148,11 @@ export function transformMap<IN = any, OUT = IN>(
144
148
  cb() // done processing
145
149
  }
146
150
  } catch (err) {
147
- console.error(err)
151
+ logger.error(err)
148
152
 
149
153
  errors++
150
154
 
151
- logErrorStats()
155
+ logErrorStats(logger)
152
156
 
153
157
  if (onError) {
154
158
  try {
@@ -172,9 +176,9 @@ export function transformMap<IN = any, OUT = IN>(
172
176
  },
173
177
  )
174
178
 
175
- function logErrorStats(final = false): void {
179
+ function logErrorStats(logger: CommonLogger, final = false): void {
176
180
  if (!errors) return
177
181
 
178
- console.log(`${metric} ${final ? 'final ' : ''}errors: ${yellow(errors)}`)
182
+ logger.log(`${metric} ${final ? 'final ' : ''}errors: ${yellow(errors)}`)
179
183
  }
180
184
  }
@@ -1,5 +1,5 @@
1
1
  import { inspect, InspectOptions } from 'util'
2
- import { StringifyAnyOptions, _stringifyAny } from '@naturalcycles/js-lib'
2
+ import { StringifyAnyOptions, _stringifyAny, JsonStringifyFunction } from '@naturalcycles/js-lib'
3
3
 
4
4
  export interface InspectAnyOptions extends StringifyAnyOptions, InspectOptions {}
5
5
 
@@ -8,6 +8,11 @@ const INSPECT_OPT: InspectOptions = {
8
8
  depth: 10, // default: 2
9
9
  }
10
10
 
11
+ /**
12
+ * Just a convenience export of a const that fulfills the JsonStringifyFunction interface.
13
+ */
14
+ export const inspectAnyStringifyFn: JsonStringifyFunction = obj => inspectAny(obj)
15
+
11
16
  /**
12
17
  * Transforms ANY to human-readable string (via util.inspect mainly).
13
18
  * Safe (no error throwing).
@@ -6,6 +6,7 @@ import {
6
6
  _filterNullishValues,
7
7
  _isObject,
8
8
  _substringBefore,
9
+ CommonLogger,
9
10
  } from '@naturalcycles/js-lib'
10
11
  import Ajv, { ValidateFunction } from 'ajv'
11
12
  import { inspectAny, requireFileToExist } from '../../index'
@@ -56,6 +57,11 @@ export interface AjvSchemaCfg {
56
57
  */
57
58
  logErrors: boolean
58
59
 
60
+ /**
61
+ * Default to `console`
62
+ */
63
+ logger: CommonLogger
64
+
59
65
  /**
60
66
  * Option of Ajv.
61
67
  * If set to true - will mutate your input objects!
@@ -76,6 +82,7 @@ export class AjvSchema<T = unknown> {
76
82
  private constructor(public schema: JsonSchema<T>, cfg: Partial<AjvSchemaCfg> = {}) {
77
83
  this.cfg = {
78
84
  logErrors: true,
85
+ logger: console,
79
86
  separator: '\n',
80
87
  ...cfg,
81
88
  ajv:
@@ -168,7 +175,7 @@ export class AjvSchema<T = unknown> {
168
175
  message = [message, 'Input: ' + strValue].join(separator)
169
176
 
170
177
  if (logErrors) {
171
- console.log(errors)
178
+ this.cfg.logger.error(errors)
172
179
  }
173
180
 
174
181
  return new AjvValidationError(