@naturalcycles/nodejs-lib 12.52.0 → 12.55.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/got/getGot.js +4 -3
- package/dist/got/got.model.d.ts +5 -1
- package/dist/index.d.ts +4 -3
- package/dist/index.js +4 -1
- package/dist/script/index.d.ts +5 -0
- package/dist/script/index.js +6 -5
- package/dist/slack/slack.service.d.ts +11 -2
- package/dist/slack/slack.service.js +15 -0
- package/dist/slack/slack.service.model.d.ts +2 -2
- package/dist/stream/ndjson/transformToNDJson.d.ts +4 -0
- package/dist/stream/ndjson/transformToNDJson.js +15 -2
- package/dist/stream/transform/beta/transformMap2.d.ts +17 -0
- package/dist/stream/transform/beta/transformMap2.js +98 -0
- package/dist/stream/transform/transformMap.d.ts +2 -1
- package/dist/stream/transform/transformMap.js +6 -6
- package/dist/string/inspectAny.d.ts +5 -1
- package/dist/string/inspectAny.js +6 -1
- package/dist/validation/ajv/ajvSchema.d.ts +5 -1
- package/dist/validation/ajv/ajvSchema.js +2 -1
- package/package.json +1 -1
- package/src/diff/tableDiff.ts +2 -2
- package/src/got/getGot.ts +6 -4
- package/src/got/got.model.ts +6 -1
- package/src/index.ts +5 -1
- package/src/script/index.ts +14 -5
- package/src/slack/slack.service.model.ts +2 -2
- package/src/slack/slack.service.ts +38 -2
- package/src/stream/ndjson/transformToNDJson.ts +21 -2
- package/src/stream/transform/beta/transformMap2.ts +134 -0
- package/src/stream/transform/transformMap.ts +9 -5
- package/src/string/inspectAny.ts +6 -1
- package/src/validation/ajv/ajvSchema.ts +8 -1
package/dist/got/getGot.js
CHANGED
|
@@ -13,6 +13,7 @@ const colors_1 = require("../colors");
|
|
|
13
13
|
* 3. Reasonable defaults(tm), e.g non-infinite Timeout
|
|
14
14
|
*/
|
|
15
15
|
function getGot(opt = {}) {
|
|
16
|
+
opt.logger || (opt.logger = console);
|
|
16
17
|
return got_1.default.extend({
|
|
17
18
|
// Most-important is to set to anything non-empty (so, requests don't "hang" by default).
|
|
18
19
|
// Should be long enough to handle for slow responses from scaled cloud APIs in times of spikes
|
|
@@ -93,7 +94,7 @@ function gotBeforeRequestHook(opt) {
|
|
|
93
94
|
};
|
|
94
95
|
if (opt.logStart) {
|
|
95
96
|
const shortUrl = getShortUrl(opt, options.url, options.prefixUrl);
|
|
96
|
-
|
|
97
|
+
opt.logger.log([(0, colors_1.dimGrey)(' >>'), (0, colors_1.dimGrey)(options.method), (0, colors_1.grey)(shortUrl)].join(' '));
|
|
97
98
|
}
|
|
98
99
|
};
|
|
99
100
|
}
|
|
@@ -104,7 +105,7 @@ function gotAfterResponseHook(opt = {}) {
|
|
|
104
105
|
const { started } = resp.request.options.context;
|
|
105
106
|
const { url, prefixUrl, method } = resp.request.options;
|
|
106
107
|
const shortUrl = getShortUrl(opt, url, prefixUrl);
|
|
107
|
-
|
|
108
|
+
opt.logger.log([
|
|
108
109
|
(0, colors_1.dimGrey)(' <<'),
|
|
109
110
|
coloredHttpCode(resp.statusCode),
|
|
110
111
|
(0, colors_1.dimGrey)(method),
|
|
@@ -117,7 +118,7 @@ function gotAfterResponseHook(opt = {}) {
|
|
|
117
118
|
}
|
|
118
119
|
// Error responses are not logged, cause they're included in Error message already
|
|
119
120
|
if (opt.logResponse && success) {
|
|
120
|
-
|
|
121
|
+
opt.logger.log((0, __1.inspectAny)(resp.body, { maxLen: opt.maxResponseLength }));
|
|
121
122
|
}
|
|
122
123
|
return resp;
|
|
123
124
|
};
|
package/dist/got/got.model.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { AnyObject } from '@naturalcycles/js-lib';
|
|
1
|
+
import { AnyObject, CommonLogger } from '@naturalcycles/js-lib';
|
|
2
2
|
import type { Options } from 'got';
|
|
3
3
|
export interface GetGotOptions extends Options {
|
|
4
4
|
/**
|
|
@@ -27,6 +27,10 @@ export interface GetGotOptions extends Options {
|
|
|
27
27
|
* Set to false to strip searchParams from url when logging (both success and error)
|
|
28
28
|
*/
|
|
29
29
|
logWithSearchParams?: boolean;
|
|
30
|
+
/**
|
|
31
|
+
* Defaults to `console`
|
|
32
|
+
*/
|
|
33
|
+
logger?: CommonLogger;
|
|
30
34
|
/**
|
|
31
35
|
* Max length of response object before it's truncated.
|
|
32
36
|
*
|
package/dist/index.d.ts
CHANGED
|
@@ -51,7 +51,7 @@ import { writableForEach } from './stream/writable/writableForEach';
|
|
|
51
51
|
import { writableFork } from './stream/writable/writableFork';
|
|
52
52
|
import { writablePushToArray } from './stream/writable/writablePushToArray';
|
|
53
53
|
import { writableVoid } from './stream/writable/writableVoid';
|
|
54
|
-
import { inspectAny, InspectAnyOptions } from './string/inspectAny';
|
|
54
|
+
import { inspectAny, InspectAnyOptions, inspectAnyStringifyFn } from './string/inspectAny';
|
|
55
55
|
import { requireEnvKeys, requireFileToExist } from './util/env.util';
|
|
56
56
|
import { LRUMemoCache } from './util/lruMemoCache';
|
|
57
57
|
import { gunzipBuffer, gunzipToString, gzipBuffer, gzipString, unzipBuffer, unzipToString, zipBuffer, zipString } from './util/zip.util';
|
|
@@ -65,5 +65,6 @@ import { anyObjectSchema, anySchema, arraySchema, oneOfSchema, binarySchema, boo
|
|
|
65
65
|
import { JoiValidationError, JoiValidationErrorData } from './validation/joi/joi.validation.error';
|
|
66
66
|
import { convert, getValidationResult, isValid, JoiValidationResult, undefinedIfInvalid, validate } from './validation/joi/joi.validation.util';
|
|
67
67
|
import { sanitizeHTML, SanitizeHTMLOptions } from './validation/sanitize.util';
|
|
68
|
-
|
|
69
|
-
export {
|
|
68
|
+
import { runScript, RunScriptOptions } from './script';
|
|
69
|
+
export type { RunScriptOptions, JoiValidationErrorData, JoiValidationResult, ValidationErrorItem, ExtendedJoi, SchemaTyped, AnySchema, AnySchemaTyped, ArraySchemaTyped, BooleanSchemaTyped, NumberSchemaTyped, ObjectSchemaTyped, StringSchemaTyped, IDebug, IDebugger, SlackServiceCfg, SlackMessage, SlackMessageProps, SlackApiBody, SlackMessagePrefixHook, ReadableTyped, WritableTyped, TransformTyped, PipelineFromNDJsonFileOptions, PipelineToNDJsonFileOptions, TransformJsonParseOptions, TransformToNDJsonOptions, TransformMapOptions, TransformMapSyncOptions, NDJSONStreamForEachOptions, TransformOptions, TransformLogProgressOptions, TransformMultiThreadedOptions, WorkerClassInterface, WorkerInput, WorkerOutput, TableDiffOptions, InspectAnyOptions, Got, GetGotOptions, AfterResponseHook, BeforeErrorHook, BeforeRequestHook, AjvValidationOptions, AjvSchemaCfg, AjvValidationErrorData, SanitizeHTMLOptions, };
|
|
70
|
+
export { JoiValidationError, validate, getValidationResult, isValid, undefinedIfInvalid, convert, Joi, booleanSchema, booleanDefaultToFalseSchema, stringSchema, numberSchema, integerSchema, percentageSchema, dateStringSchema, arraySchema, binarySchema, objectSchema, oneOfSchema, anySchema, anyObjectSchema, baseDBEntitySchema, savedDBEntitySchema, idSchema, unixTimestampSchema, verSchema, emailSchema, SEM_VER_PATTERN, semVerSchema, userAgentSchema, utcOffsetSchema, ipAddressSchema, slugSchema, urlSchema, processSharedUtil, zipBuffer, gzipBuffer, unzipBuffer, gunzipBuffer, zipString, gzipString, unzipToString, gunzipToString, requireEnvKeys, requireFileToExist, LRUMemoCache, stringId, stringIdAsync, stringIdUnsafe, ALPHABET_NUMBER, ALPHABET_LOWERCASE, ALPHABET_UPPERCASE, ALPHABET_ALPHANUMERIC_LOWERCASE, ALPHABET_ALPHANUMERIC_UPPERCASE, ALPHABET_ALPHANUMERIC, md5, hash, hashAsBuffer, md5AsBuffer, stringToBase64, base64ToString, bufferToBase64, base64ToBuffer, Debug, getSecretMap, setSecretMap, loadSecretsFromEnv, loadSecretsFromJsonFile, removeSecretsFromEnv, secret, secretOptional, memoryUsage, memoryUsageFull, SlackService, slackDefaultMessagePrefixHook, readableCreate, readableFrom, readableFromArray, readableToArray, readableForEach, readableForEachSync, readableMap, readableMapToArray, _pipeline, transformBuffer, ndjsonMap, ndJsonFileRead, ndJsonFileWrite, ndjsonStreamForEach, pipelineFromNDJsonFile, pipelineToNDJsonFile, NDJsonStats, streamToNDJsonFile, transformJsonParse, bufferReviver, transformToNDJson, transformFilter, transformFilterSync, transformMap, transformMapSync, transformMapSimple, transformNoOp, writableForEach, writablePushToArray, transformSplit, transformToString, transformToArray, transformTap, transformLogProgress, transformLimit, writableVoid, writableFork, transformMultiThreaded, BaseWorkerClass, tableDiff, inspectAny, inspectAnyStringifyFn, getGot, HTTPError, TimeoutError, _chunkBuffer, Ajv, getAjv, AjvSchema, AjvValidationError, readJsonSchemas, readAjvSchemas, hasColors, sanitizeHTML, runScript, };
|
package/dist/index.js
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.ALPHABET_LOWERCASE = exports.ALPHABET_NUMBER = exports.stringIdUnsafe = exports.stringIdAsync = exports.stringId = exports.LRUMemoCache = exports.requireFileToExist = exports.requireEnvKeys = exports.gunzipToString = exports.unzipToString = exports.gzipString = exports.zipString = exports.gunzipBuffer = exports.unzipBuffer = exports.gzipBuffer = exports.zipBuffer = exports.processSharedUtil = exports.urlSchema = exports.slugSchema = exports.ipAddressSchema = exports.utcOffsetSchema = exports.userAgentSchema = exports.semVerSchema = exports.SEM_VER_PATTERN = exports.emailSchema = exports.verSchema = exports.unixTimestampSchema = exports.idSchema = exports.savedDBEntitySchema = exports.baseDBEntitySchema = exports.anyObjectSchema = exports.anySchema = exports.oneOfSchema = exports.objectSchema = exports.binarySchema = exports.arraySchema = exports.dateStringSchema = exports.percentageSchema = exports.integerSchema = exports.numberSchema = exports.stringSchema = exports.booleanDefaultToFalseSchema = exports.booleanSchema = exports.Joi = exports.convert = exports.undefinedIfInvalid = exports.isValid = exports.getValidationResult = exports.validate = exports.JoiValidationError = void 0;
|
|
4
4
|
exports.transformMapSimple = exports.transformMapSync = exports.transformMap = exports.transformFilterSync = exports.transformFilter = exports.transformToNDJson = exports.bufferReviver = exports.transformJsonParse = exports.streamToNDJsonFile = exports.NDJsonStats = exports.pipelineToNDJsonFile = exports.pipelineFromNDJsonFile = exports.ndjsonStreamForEach = exports.ndJsonFileWrite = exports.ndJsonFileRead = exports.ndjsonMap = exports.transformBuffer = exports._pipeline = exports.readableMapToArray = exports.readableMap = exports.readableForEachSync = exports.readableForEach = exports.readableToArray = exports.readableFromArray = exports.readableFrom = exports.readableCreate = exports.slackDefaultMessagePrefixHook = exports.SlackService = exports.memoryUsageFull = exports.memoryUsage = exports.secretOptional = exports.secret = exports.removeSecretsFromEnv = exports.loadSecretsFromJsonFile = exports.loadSecretsFromEnv = exports.setSecretMap = exports.getSecretMap = exports.Debug = exports.base64ToBuffer = exports.bufferToBase64 = exports.base64ToString = exports.stringToBase64 = exports.md5AsBuffer = exports.hashAsBuffer = exports.hash = exports.md5 = exports.ALPHABET_ALPHANUMERIC = exports.ALPHABET_ALPHANUMERIC_UPPERCASE = exports.ALPHABET_ALPHANUMERIC_LOWERCASE = exports.ALPHABET_UPPERCASE = void 0;
|
|
5
|
-
exports.sanitizeHTML = exports.hasColors = exports.readAjvSchemas = exports.readJsonSchemas = exports.AjvValidationError = exports.AjvSchema = exports.getAjv = exports.Ajv = exports._chunkBuffer = exports.TimeoutError = exports.HTTPError = exports.getGot = exports.inspectAny = exports.tableDiff = exports.BaseWorkerClass = exports.transformMultiThreaded = exports.writableFork = exports.writableVoid = exports.transformLimit = exports.transformLogProgress = exports.transformTap = exports.transformToArray = exports.transformToString = exports.transformSplit = exports.writablePushToArray = exports.writableForEach = exports.transformNoOp = void 0;
|
|
5
|
+
exports.runScript = exports.sanitizeHTML = exports.hasColors = exports.readAjvSchemas = exports.readJsonSchemas = exports.AjvValidationError = exports.AjvSchema = exports.getAjv = exports.Ajv = exports._chunkBuffer = exports.TimeoutError = exports.HTTPError = exports.getGot = exports.inspectAnyStringifyFn = exports.inspectAny = exports.tableDiff = exports.BaseWorkerClass = exports.transformMultiThreaded = exports.writableFork = exports.writableVoid = exports.transformLimit = exports.transformLogProgress = exports.transformTap = exports.transformToArray = exports.transformToString = exports.transformSplit = exports.writablePushToArray = exports.writableForEach = exports.transformNoOp = void 0;
|
|
6
6
|
const ajv_1 = require("ajv");
|
|
7
7
|
exports.Ajv = ajv_1.default;
|
|
8
8
|
const got_1 = require("got");
|
|
@@ -128,6 +128,7 @@ const writableVoid_1 = require("./stream/writable/writableVoid");
|
|
|
128
128
|
Object.defineProperty(exports, "writableVoid", { enumerable: true, get: function () { return writableVoid_1.writableVoid; } });
|
|
129
129
|
const inspectAny_1 = require("./string/inspectAny");
|
|
130
130
|
Object.defineProperty(exports, "inspectAny", { enumerable: true, get: function () { return inspectAny_1.inspectAny; } });
|
|
131
|
+
Object.defineProperty(exports, "inspectAnyStringifyFn", { enumerable: true, get: function () { return inspectAny_1.inspectAnyStringifyFn; } });
|
|
131
132
|
const env_util_1 = require("./util/env.util");
|
|
132
133
|
Object.defineProperty(exports, "requireEnvKeys", { enumerable: true, get: function () { return env_util_1.requireEnvKeys; } });
|
|
133
134
|
Object.defineProperty(exports, "requireFileToExist", { enumerable: true, get: function () { return env_util_1.requireFileToExist; } });
|
|
@@ -190,3 +191,5 @@ Object.defineProperty(exports, "undefinedIfInvalid", { enumerable: true, get: fu
|
|
|
190
191
|
Object.defineProperty(exports, "validate", { enumerable: true, get: function () { return joi_validation_util_1.validate; } });
|
|
191
192
|
const sanitize_util_1 = require("./validation/sanitize.util");
|
|
192
193
|
Object.defineProperty(exports, "sanitizeHTML", { enumerable: true, get: function () { return sanitize_util_1.sanitizeHTML; } });
|
|
194
|
+
const script_1 = require("./script");
|
|
195
|
+
Object.defineProperty(exports, "runScript", { enumerable: true, get: function () { return script_1.runScript; } });
|
package/dist/script/index.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { CommonLogger } from '@naturalcycles/js-lib';
|
|
1
2
|
export interface RunScriptOptions {
|
|
2
3
|
/**
|
|
3
4
|
* @default false
|
|
@@ -5,6 +6,10 @@ export interface RunScriptOptions {
|
|
|
5
6
|
* Currently it exists because of `jest --maxWorkers=1` behavior. To be investigated more..
|
|
6
7
|
*/
|
|
7
8
|
noExit?: boolean;
|
|
9
|
+
/**
|
|
10
|
+
* Default to `console`
|
|
11
|
+
*/
|
|
12
|
+
logger?: CommonLogger;
|
|
8
13
|
}
|
|
9
14
|
/**
|
|
10
15
|
* Use it in your top-level scripts like this:
|
package/dist/script/index.js
CHANGED
|
@@ -18,23 +18,24 @@ exports.runScript = void 0;
|
|
|
18
18
|
* This function is kept light, dependency-free, exported separately.
|
|
19
19
|
*/
|
|
20
20
|
function runScript(fn, opt = {}) {
|
|
21
|
+
const { logger = console, noExit } = opt;
|
|
21
22
|
process.on('uncaughtException', err => {
|
|
22
|
-
|
|
23
|
+
logger.error('uncaughtException:', err);
|
|
23
24
|
});
|
|
24
25
|
process.on('unhandledRejection', err => {
|
|
25
|
-
|
|
26
|
+
logger.error('unhandledRejection:', err);
|
|
26
27
|
});
|
|
27
28
|
void (async () => {
|
|
28
29
|
try {
|
|
29
30
|
await fn();
|
|
30
|
-
if (!
|
|
31
|
+
if (!noExit) {
|
|
31
32
|
setImmediate(() => process.exit(0));
|
|
32
33
|
}
|
|
33
34
|
}
|
|
34
35
|
catch (err) {
|
|
35
|
-
|
|
36
|
+
logger.error('runScript error:', err);
|
|
36
37
|
process.exitCode = 1;
|
|
37
|
-
if (!
|
|
38
|
+
if (!noExit) {
|
|
38
39
|
setImmediate(() => process.exit(1));
|
|
39
40
|
}
|
|
40
41
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { AnyObject, CommonLogger, CommonLogLevel } from '@naturalcycles/js-lib';
|
|
2
2
|
import { SlackAttachmentField, SlackMessage, SlackServiceCfg } from './slack.service.model';
|
|
3
3
|
/**
|
|
4
4
|
* Has 2 main methods:
|
|
@@ -20,6 +20,15 @@ export declare class SlackService<CTX = any> {
|
|
|
20
20
|
*/
|
|
21
21
|
log(...items: any[]): Promise<void>;
|
|
22
22
|
send(input: SlackMessage<CTX> | string, ctx?: CTX): Promise<void>;
|
|
23
|
-
kvToFields(kv:
|
|
23
|
+
kvToFields(kv: AnyObject): SlackAttachmentField[];
|
|
24
|
+
/**
|
|
25
|
+
* Returns a CommonLogger implementation based on this SlackService instance.
|
|
26
|
+
*/
|
|
27
|
+
getCommonLogger(opt: {
|
|
28
|
+
minLogLevel: CommonLogLevel;
|
|
29
|
+
logChannel?: string;
|
|
30
|
+
warnChannel?: string;
|
|
31
|
+
errorChannel?: string;
|
|
32
|
+
}): CommonLogger;
|
|
24
33
|
}
|
|
25
34
|
export declare function slackDefaultMessagePrefixHook(msg: SlackMessage): string[];
|
|
@@ -110,6 +110,21 @@ class SlackService {
|
|
|
110
110
|
short: String(v).length < 80,
|
|
111
111
|
}));
|
|
112
112
|
}
|
|
113
|
+
/**
|
|
114
|
+
* Returns a CommonLogger implementation based on this SlackService instance.
|
|
115
|
+
*/
|
|
116
|
+
getCommonLogger(opt) {
|
|
117
|
+
const { minLogLevel = 'log', logChannel, warnChannel, errorChannel } = opt;
|
|
118
|
+
const defaultChannel = this.cfg.defaults?.channel || DEFAULTS.channel;
|
|
119
|
+
const q = new js_lib_1.PQueue({
|
|
120
|
+
concurrency: 1,
|
|
121
|
+
});
|
|
122
|
+
return (0, js_lib_1.commonLoggerMinLevel)({
|
|
123
|
+
log: (...args) => q.push(() => this.send({ items: args, channel: logChannel || defaultChannel })),
|
|
124
|
+
warn: (...args) => q.push(() => this.send({ items: args, channel: warnChannel || defaultChannel })),
|
|
125
|
+
error: (...args) => q.push(() => this.send({ items: args, channel: errorChannel || defaultChannel })),
|
|
126
|
+
}, minLogLevel);
|
|
127
|
+
}
|
|
113
128
|
}
|
|
114
129
|
exports.SlackService = SlackService;
|
|
115
130
|
function slackDefaultMessagePrefixHook(msg) {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { AnyObject, CommonLogger } from '@naturalcycles/js-lib';
|
|
2
2
|
import { InspectAnyOptions } from '..';
|
|
3
3
|
/**
|
|
4
4
|
* Properties that exists both in SlackApiBody (as per Slack API) and SlackMessage (our abstraction).
|
|
@@ -39,7 +39,7 @@ export interface SlackMessage<CTX = any> extends SlackMessageProps {
|
|
|
39
39
|
/**
|
|
40
40
|
* Keys-values will be rendered as MessageAttachment with Fields
|
|
41
41
|
*/
|
|
42
|
-
kv?:
|
|
42
|
+
kv?: AnyObject;
|
|
43
43
|
/**
|
|
44
44
|
* If specified - adds @name1, @name2 in the end of the message
|
|
45
45
|
*/
|
|
@@ -16,6 +16,10 @@ export interface TransformToNDJsonOptions {
|
|
|
16
16
|
* @default `\n`
|
|
17
17
|
*/
|
|
18
18
|
separator?: string;
|
|
19
|
+
/**
|
|
20
|
+
* @experimental
|
|
21
|
+
*/
|
|
22
|
+
useFlatstr?: boolean;
|
|
19
23
|
}
|
|
20
24
|
/**
|
|
21
25
|
* Transforms objects (objectMode=true) into chunks \n-terminated JSON strings (readableObjectMode=false).
|
|
@@ -7,7 +7,7 @@ const js_lib_1 = require("@naturalcycles/js-lib");
|
|
|
7
7
|
* Transforms objects (objectMode=true) into chunks \n-terminated JSON strings (readableObjectMode=false).
|
|
8
8
|
*/
|
|
9
9
|
function transformToNDJson(opt = {}) {
|
|
10
|
-
const { strict = true, separator = '\n', sortObjects = false } = opt;
|
|
10
|
+
const { strict = true, separator = '\n', sortObjects = false, useFlatstr = false } = opt;
|
|
11
11
|
return new stream_1.Transform({
|
|
12
12
|
objectMode: true,
|
|
13
13
|
readableObjectMode: false,
|
|
@@ -16,7 +16,12 @@ function transformToNDJson(opt = {}) {
|
|
|
16
16
|
if (sortObjects) {
|
|
17
17
|
chunk = (0, js_lib_1._sortObjectDeep)(chunk);
|
|
18
18
|
}
|
|
19
|
-
|
|
19
|
+
if (useFlatstr) {
|
|
20
|
+
cb(null, flatstr(JSON.stringify(chunk) + separator));
|
|
21
|
+
}
|
|
22
|
+
else {
|
|
23
|
+
cb(null, JSON.stringify(chunk) + separator);
|
|
24
|
+
}
|
|
20
25
|
}
|
|
21
26
|
catch (err) {
|
|
22
27
|
console.error(err);
|
|
@@ -31,3 +36,11 @@ function transformToNDJson(opt = {}) {
|
|
|
31
36
|
});
|
|
32
37
|
}
|
|
33
38
|
exports.transformToNDJson = transformToNDJson;
|
|
39
|
+
/**
|
|
40
|
+
* Based on: https://github.com/davidmarkclements/flatstr/blob/master/index.js
|
|
41
|
+
*/
|
|
42
|
+
function flatstr(s) {
|
|
43
|
+
// eslint-disable-next-line
|
|
44
|
+
s | 0;
|
|
45
|
+
return s;
|
|
46
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { AsyncMapper } from '@naturalcycles/js-lib';
|
|
2
|
+
import { TransformTyped } from '../../stream.model';
|
|
3
|
+
import { TransformMapOptions } from '../transformMap';
|
|
4
|
+
export declare function notNullishPredicate(item: any): boolean;
|
|
5
|
+
/**
|
|
6
|
+
* Like pMap, but for streams.
|
|
7
|
+
* Inspired by `through2`.
|
|
8
|
+
* Main feature is concurrency control (implemented via `through2-concurrent`) and convenient options.
|
|
9
|
+
* Using this allows native stream .pipe() to work and use backpressure.
|
|
10
|
+
*
|
|
11
|
+
* Only works in objectMode (due to through2Concurrent).
|
|
12
|
+
*
|
|
13
|
+
* Concurrency defaults to 16.
|
|
14
|
+
*
|
|
15
|
+
* If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
|
|
16
|
+
*/
|
|
17
|
+
export declare function transformMap2<IN = any, OUT = IN>(mapper: AsyncMapper<IN, OUT>, opt?: TransformMapOptions<IN, OUT>): TransformTyped<IN, OUT>;
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.transformMap2 = exports.notNullishPredicate = void 0;
|
|
4
|
+
const stream_1 = require("stream");
|
|
5
|
+
const js_lib_1 = require("@naturalcycles/js-lib");
|
|
6
|
+
const colors_1 = require("../../../colors");
|
|
7
|
+
function notNullishPredicate(item) {
|
|
8
|
+
return item !== undefined && item !== null;
|
|
9
|
+
}
|
|
10
|
+
exports.notNullishPredicate = notNullishPredicate;
|
|
11
|
+
/**
|
|
12
|
+
* Like pMap, but for streams.
|
|
13
|
+
* Inspired by `through2`.
|
|
14
|
+
* Main feature is concurrency control (implemented via `through2-concurrent`) and convenient options.
|
|
15
|
+
* Using this allows native stream .pipe() to work and use backpressure.
|
|
16
|
+
*
|
|
17
|
+
* Only works in objectMode (due to through2Concurrent).
|
|
18
|
+
*
|
|
19
|
+
* Concurrency defaults to 16.
|
|
20
|
+
*
|
|
21
|
+
* If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
|
|
22
|
+
*/
|
|
23
|
+
function transformMap2(mapper, opt = {}) {
|
|
24
|
+
const { concurrency = 16, predicate = notNullishPredicate, errorMode = js_lib_1.ErrorMode.THROW_IMMEDIATELY, flattenArrayOutput, onError, beforeFinal, metric = 'stream', logger = console, } = opt;
|
|
25
|
+
let index = -1;
|
|
26
|
+
let isRejected = false;
|
|
27
|
+
let errors = 0;
|
|
28
|
+
const collectedErrors = []; // only used if errorMode == THROW_AGGREGATED
|
|
29
|
+
const q = new js_lib_1.PQueue({
|
|
30
|
+
concurrency,
|
|
31
|
+
resolveOn: 'start',
|
|
32
|
+
// debug: true,
|
|
33
|
+
});
|
|
34
|
+
return new stream_1.Transform({
|
|
35
|
+
objectMode: true,
|
|
36
|
+
async final(cb) {
|
|
37
|
+
// console.log('transformMap final', {index}, q.inFlight, q.queueSize)
|
|
38
|
+
// wait for the current inFlight jobs to complete and push their results
|
|
39
|
+
await q.onIdle();
|
|
40
|
+
logErrorStats(logger, true);
|
|
41
|
+
await beforeFinal?.(); // call beforeFinal if defined
|
|
42
|
+
if (collectedErrors.length) {
|
|
43
|
+
// emit Aggregated error
|
|
44
|
+
cb(new js_lib_1.AggregatedError(collectedErrors));
|
|
45
|
+
}
|
|
46
|
+
else {
|
|
47
|
+
// emit no error
|
|
48
|
+
cb();
|
|
49
|
+
}
|
|
50
|
+
},
|
|
51
|
+
async transform(chunk, _encoding, cb) {
|
|
52
|
+
index++;
|
|
53
|
+
// console.log('transform', {index})
|
|
54
|
+
// Stop processing if THROW_IMMEDIATELY mode is used
|
|
55
|
+
if (isRejected && errorMode === js_lib_1.ErrorMode.THROW_IMMEDIATELY)
|
|
56
|
+
return cb();
|
|
57
|
+
// It resolves when it is successfully STARTED execution.
|
|
58
|
+
// If it's queued instead - it'll wait and resolve only upon START.
|
|
59
|
+
await q.push(async () => {
|
|
60
|
+
try {
|
|
61
|
+
const currentIndex = index; // because we need to pass it to 2 functions - mapper and predicate. Refers to INPUT index (since it may return multiple outputs)
|
|
62
|
+
const res = await mapper(chunk, currentIndex);
|
|
63
|
+
const passedResults = await (0, js_lib_1.pFilter)(flattenArrayOutput && Array.isArray(res) ? res : [res], async (r) => await predicate(r, currentIndex));
|
|
64
|
+
passedResults.forEach(r => this.push(r));
|
|
65
|
+
}
|
|
66
|
+
catch (err) {
|
|
67
|
+
logger.error(err);
|
|
68
|
+
errors++;
|
|
69
|
+
logErrorStats(logger);
|
|
70
|
+
if (onError) {
|
|
71
|
+
try {
|
|
72
|
+
onError(err, chunk);
|
|
73
|
+
}
|
|
74
|
+
catch { }
|
|
75
|
+
}
|
|
76
|
+
if (errorMode === js_lib_1.ErrorMode.THROW_IMMEDIATELY) {
|
|
77
|
+
isRejected = true;
|
|
78
|
+
// Emit error immediately
|
|
79
|
+
// return cb(err as Error)
|
|
80
|
+
return this.emit('error', err);
|
|
81
|
+
}
|
|
82
|
+
if (errorMode === js_lib_1.ErrorMode.THROW_AGGREGATED) {
|
|
83
|
+
collectedErrors.push(err);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
});
|
|
87
|
+
// Resolved, which means it STARTED processing
|
|
88
|
+
// This means we can take more load
|
|
89
|
+
cb();
|
|
90
|
+
},
|
|
91
|
+
});
|
|
92
|
+
function logErrorStats(logger, final = false) {
|
|
93
|
+
if (!errors)
|
|
94
|
+
return;
|
|
95
|
+
logger.log(`${metric} ${final ? 'final ' : ''}errors: ${(0, colors_1.yellow)(errors)}`);
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
exports.transformMap2 = transformMap2;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { AsyncMapper, AsyncPredicate, ErrorMode } from '@naturalcycles/js-lib';
|
|
1
|
+
import { AsyncMapper, AsyncPredicate, CommonLogger, ErrorMode } from '@naturalcycles/js-lib';
|
|
2
2
|
import { TransformTyped } from '../stream.model';
|
|
3
3
|
export interface TransformMapOptions<IN = any, OUT = IN> {
|
|
4
4
|
/**
|
|
@@ -41,6 +41,7 @@ export interface TransformMapOptions<IN = any, OUT = IN> {
|
|
|
41
41
|
* If defined - called BEFORE `final()` callback is called.
|
|
42
42
|
*/
|
|
43
43
|
beforeFinal?: () => any;
|
|
44
|
+
logger?: CommonLogger;
|
|
44
45
|
}
|
|
45
46
|
export declare function notNullishPredicate(item: any): boolean;
|
|
46
47
|
/**
|
|
@@ -21,7 +21,7 @@ exports.notNullishPredicate = notNullishPredicate;
|
|
|
21
21
|
* If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
|
|
22
22
|
*/
|
|
23
23
|
function transformMap(mapper, opt = {}) {
|
|
24
|
-
const { concurrency = 16, predicate = notNullishPredicate, errorMode = js_lib_1.ErrorMode.THROW_IMMEDIATELY, flattenArrayOutput, onError, beforeFinal, metric = 'stream', } = opt;
|
|
24
|
+
const { concurrency = 16, predicate = notNullishPredicate, errorMode = js_lib_1.ErrorMode.THROW_IMMEDIATELY, flattenArrayOutput, onError, beforeFinal, metric = 'stream', logger = console, } = opt;
|
|
25
25
|
let index = -1;
|
|
26
26
|
let isRejected = false;
|
|
27
27
|
let errors = 0;
|
|
@@ -31,7 +31,7 @@ function transformMap(mapper, opt = {}) {
|
|
|
31
31
|
// autoDestroy: true,
|
|
32
32
|
async final(cb) {
|
|
33
33
|
// console.log('transformMap final')
|
|
34
|
-
logErrorStats(true);
|
|
34
|
+
logErrorStats(logger, true);
|
|
35
35
|
await beforeFinal?.(); // call beforeFinal if defined
|
|
36
36
|
if (collectedErrors.length) {
|
|
37
37
|
// emit Aggregated error
|
|
@@ -64,9 +64,9 @@ function transformMap(mapper, opt = {}) {
|
|
|
64
64
|
}
|
|
65
65
|
}
|
|
66
66
|
catch (err) {
|
|
67
|
-
|
|
67
|
+
logger.error(err);
|
|
68
68
|
errors++;
|
|
69
|
-
logErrorStats();
|
|
69
|
+
logErrorStats(logger);
|
|
70
70
|
if (onError) {
|
|
71
71
|
try {
|
|
72
72
|
onError(err, chunk);
|
|
@@ -85,10 +85,10 @@ function transformMap(mapper, opt = {}) {
|
|
|
85
85
|
cb();
|
|
86
86
|
}
|
|
87
87
|
});
|
|
88
|
-
function logErrorStats(final = false) {
|
|
88
|
+
function logErrorStats(logger, final = false) {
|
|
89
89
|
if (!errors)
|
|
90
90
|
return;
|
|
91
|
-
|
|
91
|
+
logger.log(`${metric} ${final ? 'final ' : ''}errors: ${(0, colors_1.yellow)(errors)}`);
|
|
92
92
|
}
|
|
93
93
|
}
|
|
94
94
|
exports.transformMap = transformMap;
|
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
2
|
import { InspectOptions } from 'util';
|
|
3
|
-
import { StringifyAnyOptions } from '@naturalcycles/js-lib';
|
|
3
|
+
import { StringifyAnyOptions, JsonStringifyFunction } from '@naturalcycles/js-lib';
|
|
4
4
|
export interface InspectAnyOptions extends StringifyAnyOptions, InspectOptions {
|
|
5
5
|
}
|
|
6
|
+
/**
|
|
7
|
+
* Just a convenience export of a const that fulfills the JsonStringifyFunction interface.
|
|
8
|
+
*/
|
|
9
|
+
export declare const inspectAnyStringifyFn: JsonStringifyFunction;
|
|
6
10
|
/**
|
|
7
11
|
* Transforms ANY to human-readable string (via util.inspect mainly).
|
|
8
12
|
* Safe (no error throwing).
|
|
@@ -1,12 +1,17 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.inspectAny = void 0;
|
|
3
|
+
exports.inspectAny = exports.inspectAnyStringifyFn = void 0;
|
|
4
4
|
const util_1 = require("util");
|
|
5
5
|
const js_lib_1 = require("@naturalcycles/js-lib");
|
|
6
6
|
const INSPECT_OPT = {
|
|
7
7
|
breakLength: 80,
|
|
8
8
|
depth: 10, // default: 2
|
|
9
9
|
};
|
|
10
|
+
/**
|
|
11
|
+
* Just a convenience export of a const that fulfills the JsonStringifyFunction interface.
|
|
12
|
+
*/
|
|
13
|
+
const inspectAnyStringifyFn = obj => inspectAny(obj);
|
|
14
|
+
exports.inspectAnyStringifyFn = inspectAnyStringifyFn;
|
|
10
15
|
/**
|
|
11
16
|
* Transforms ANY to human-readable string (via util.inspect mainly).
|
|
12
17
|
* Safe (no error throwing).
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { JsonSchema, JsonSchemaBuilder } from '@naturalcycles/js-lib';
|
|
1
|
+
import { JsonSchema, JsonSchemaBuilder, CommonLogger } from '@naturalcycles/js-lib';
|
|
2
2
|
import Ajv from 'ajv';
|
|
3
3
|
import { AjvValidationError } from './ajvValidationError';
|
|
4
4
|
export interface AjvValidationOptions {
|
|
@@ -37,6 +37,10 @@ export interface AjvSchemaCfg {
|
|
|
37
37
|
* @default true
|
|
38
38
|
*/
|
|
39
39
|
logErrors: boolean;
|
|
40
|
+
/**
|
|
41
|
+
* Default to `console`
|
|
42
|
+
*/
|
|
43
|
+
logger: CommonLogger;
|
|
40
44
|
/**
|
|
41
45
|
* Option of Ajv.
|
|
42
46
|
* If set to true - will mutate your input objects!
|
|
@@ -17,6 +17,7 @@ class AjvSchema {
|
|
|
17
17
|
this.schema = schema;
|
|
18
18
|
this.cfg = {
|
|
19
19
|
logErrors: true,
|
|
20
|
+
logger: console,
|
|
20
21
|
separator: '\n',
|
|
21
22
|
...cfg,
|
|
22
23
|
ajv: cfg.ajv ||
|
|
@@ -89,7 +90,7 @@ class AjvSchema {
|
|
|
89
90
|
const strValue = (0, index_1.inspectAny)(obj, { maxLen: 1000 });
|
|
90
91
|
message = [message, 'Input: ' + strValue].join(separator);
|
|
91
92
|
if (logErrors) {
|
|
92
|
-
|
|
93
|
+
this.cfg.logger.error(errors);
|
|
93
94
|
}
|
|
94
95
|
return new ajvValidationError_1.AjvValidationError(message, (0, js_lib_1._filterNullishValues)({
|
|
95
96
|
errors,
|
package/package.json
CHANGED
package/src/diff/tableDiff.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { _truncate, AnyObject } from '@naturalcycles/js-lib'
|
|
2
2
|
|
|
3
3
|
export interface TableDiffOptions {
|
|
4
4
|
/**
|
|
@@ -37,7 +37,7 @@ export interface TableDiffOptions {
|
|
|
37
37
|
*/
|
|
38
38
|
export function tableDiff(a: AnyObject, b: AnyObject, opt: TableDiffOptions = {}): void {
|
|
39
39
|
const { maxFieldLen, aTitle = 'a', bTitle = 'b' } = opt
|
|
40
|
-
const diff:
|
|
40
|
+
const diff: AnyObject = {}
|
|
41
41
|
|
|
42
42
|
if (a && b && a !== b) {
|
|
43
43
|
new Set([...Object.keys(a), ...Object.keys(b)]).forEach(k => {
|
package/src/got/getGot.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { URL } from 'url'
|
|
2
|
-
import {
|
|
2
|
+
import { _since } from '@naturalcycles/js-lib'
|
|
3
3
|
import got, { AfterResponseHook, BeforeErrorHook, BeforeRequestHook, Got, HTTPError } from 'got'
|
|
4
4
|
import { inspectAny } from '..'
|
|
5
5
|
import { dimGrey, grey, red, yellow } from '../colors'
|
|
@@ -13,6 +13,8 @@ import { GetGotOptions, GotRequestContext } from './got.model'
|
|
|
13
13
|
* 3. Reasonable defaults(tm), e.g non-infinite Timeout
|
|
14
14
|
*/
|
|
15
15
|
export function getGot(opt: GetGotOptions = {}): Got {
|
|
16
|
+
opt.logger ||= console
|
|
17
|
+
|
|
16
18
|
return got.extend({
|
|
17
19
|
// Most-important is to set to anything non-empty (so, requests don't "hang" by default).
|
|
18
20
|
// Should be long enough to handle for slow responses from scaled cloud APIs in times of spikes
|
|
@@ -99,7 +101,7 @@ function gotBeforeRequestHook(opt: GetGotOptions): BeforeRequestHook {
|
|
|
99
101
|
|
|
100
102
|
if (opt.logStart) {
|
|
101
103
|
const shortUrl = getShortUrl(opt, options.url, options.prefixUrl)
|
|
102
|
-
|
|
104
|
+
opt.logger!.log([dimGrey(' >>'), dimGrey(options.method), grey(shortUrl)].join(' '))
|
|
103
105
|
}
|
|
104
106
|
}
|
|
105
107
|
}
|
|
@@ -113,7 +115,7 @@ function gotAfterResponseHook(opt: GetGotOptions = {}): AfterResponseHook {
|
|
|
113
115
|
const { url, prefixUrl, method } = resp.request.options
|
|
114
116
|
const shortUrl = getShortUrl(opt, url, prefixUrl)
|
|
115
117
|
|
|
116
|
-
|
|
118
|
+
opt.logger!.log(
|
|
117
119
|
[
|
|
118
120
|
dimGrey(' <<'),
|
|
119
121
|
coloredHttpCode(resp.statusCode),
|
|
@@ -129,7 +131,7 @@ function gotAfterResponseHook(opt: GetGotOptions = {}): AfterResponseHook {
|
|
|
129
131
|
|
|
130
132
|
// Error responses are not logged, cause they're included in Error message already
|
|
131
133
|
if (opt.logResponse && success) {
|
|
132
|
-
|
|
134
|
+
opt.logger!.log(inspectAny(resp.body, { maxLen: opt.maxResponseLength }))
|
|
133
135
|
}
|
|
134
136
|
|
|
135
137
|
return resp
|
package/src/got/got.model.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { AnyObject } from '@naturalcycles/js-lib'
|
|
1
|
+
import { AnyObject, CommonLogger } from '@naturalcycles/js-lib'
|
|
2
2
|
import type { Options } from 'got'
|
|
3
3
|
|
|
4
4
|
export interface GetGotOptions extends Options {
|
|
@@ -33,6 +33,11 @@ export interface GetGotOptions extends Options {
|
|
|
33
33
|
*/
|
|
34
34
|
logWithSearchParams?: boolean
|
|
35
35
|
|
|
36
|
+
/**
|
|
37
|
+
* Defaults to `console`
|
|
38
|
+
*/
|
|
39
|
+
logger?: CommonLogger
|
|
40
|
+
|
|
36
41
|
/**
|
|
37
42
|
* Max length of response object before it's truncated.
|
|
38
43
|
*
|
package/src/index.ts
CHANGED
|
@@ -108,7 +108,7 @@ import { writableForEach } from './stream/writable/writableForEach'
|
|
|
108
108
|
import { writableFork } from './stream/writable/writableFork'
|
|
109
109
|
import { writablePushToArray } from './stream/writable/writablePushToArray'
|
|
110
110
|
import { writableVoid } from './stream/writable/writableVoid'
|
|
111
|
-
import { inspectAny, InspectAnyOptions } from './string/inspectAny'
|
|
111
|
+
import { inspectAny, InspectAnyOptions, inspectAnyStringifyFn } from './string/inspectAny'
|
|
112
112
|
import { requireEnvKeys, requireFileToExist } from './util/env.util'
|
|
113
113
|
import { LRUMemoCache } from './util/lruMemoCache'
|
|
114
114
|
import {
|
|
@@ -173,8 +173,10 @@ import {
|
|
|
173
173
|
validate,
|
|
174
174
|
} from './validation/joi/joi.validation.util'
|
|
175
175
|
import { sanitizeHTML, SanitizeHTMLOptions } from './validation/sanitize.util'
|
|
176
|
+
import { runScript, RunScriptOptions } from './script'
|
|
176
177
|
|
|
177
178
|
export type {
|
|
179
|
+
RunScriptOptions,
|
|
178
180
|
JoiValidationErrorData,
|
|
179
181
|
JoiValidationResult,
|
|
180
182
|
ValidationErrorItem,
|
|
@@ -339,6 +341,7 @@ export {
|
|
|
339
341
|
BaseWorkerClass,
|
|
340
342
|
tableDiff,
|
|
341
343
|
inspectAny,
|
|
344
|
+
inspectAnyStringifyFn,
|
|
342
345
|
getGot,
|
|
343
346
|
HTTPError,
|
|
344
347
|
TimeoutError,
|
|
@@ -351,4 +354,5 @@ export {
|
|
|
351
354
|
readAjvSchemas,
|
|
352
355
|
hasColors,
|
|
353
356
|
sanitizeHTML,
|
|
357
|
+
runScript,
|
|
354
358
|
}
|
package/src/script/index.ts
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import type { CommonLogger } from '@naturalcycles/js-lib'
|
|
2
|
+
|
|
1
3
|
export interface RunScriptOptions {
|
|
2
4
|
/**
|
|
3
5
|
* @default false
|
|
@@ -5,6 +7,11 @@ export interface RunScriptOptions {
|
|
|
5
7
|
* Currently it exists because of `jest --maxWorkers=1` behavior. To be investigated more..
|
|
6
8
|
*/
|
|
7
9
|
noExit?: boolean
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Default to `console`
|
|
13
|
+
*/
|
|
14
|
+
logger?: CommonLogger
|
|
8
15
|
}
|
|
9
16
|
|
|
10
17
|
/**
|
|
@@ -24,24 +31,26 @@ export interface RunScriptOptions {
|
|
|
24
31
|
* This function is kept light, dependency-free, exported separately.
|
|
25
32
|
*/
|
|
26
33
|
export function runScript(fn: (...args: any[]) => any, opt: RunScriptOptions = {}): void {
|
|
34
|
+
const { logger = console, noExit } = opt
|
|
35
|
+
|
|
27
36
|
process.on('uncaughtException', err => {
|
|
28
|
-
|
|
37
|
+
logger.error('uncaughtException:', err)
|
|
29
38
|
})
|
|
30
39
|
process.on('unhandledRejection', err => {
|
|
31
|
-
|
|
40
|
+
logger.error('unhandledRejection:', err)
|
|
32
41
|
})
|
|
33
42
|
|
|
34
43
|
void (async () => {
|
|
35
44
|
try {
|
|
36
45
|
await fn()
|
|
37
46
|
|
|
38
|
-
if (!
|
|
47
|
+
if (!noExit) {
|
|
39
48
|
setImmediate(() => process.exit(0))
|
|
40
49
|
}
|
|
41
50
|
} catch (err) {
|
|
42
|
-
|
|
51
|
+
logger.error('runScript error:', err)
|
|
43
52
|
process.exitCode = 1
|
|
44
|
-
if (!
|
|
53
|
+
if (!noExit) {
|
|
45
54
|
setImmediate(() => process.exit(1))
|
|
46
55
|
}
|
|
47
56
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { AnyObject, CommonLogger } from '@naturalcycles/js-lib'
|
|
2
2
|
import { InspectAnyOptions } from '..'
|
|
3
3
|
|
|
4
4
|
/**
|
|
@@ -47,7 +47,7 @@ export interface SlackMessage<CTX = any> extends SlackMessageProps {
|
|
|
47
47
|
/**
|
|
48
48
|
* Keys-values will be rendered as MessageAttachment with Fields
|
|
49
49
|
*/
|
|
50
|
-
kv?:
|
|
50
|
+
kv?: AnyObject
|
|
51
51
|
|
|
52
52
|
/**
|
|
53
53
|
* If specified - adds @name1, @name2 in the end of the message
|
|
@@ -1,4 +1,11 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
_omit,
|
|
3
|
+
AnyObject,
|
|
4
|
+
CommonLogger,
|
|
5
|
+
commonLoggerMinLevel,
|
|
6
|
+
CommonLogLevel,
|
|
7
|
+
PQueue,
|
|
8
|
+
} from '@naturalcycles/js-lib'
|
|
2
9
|
import { dayjs } from '@naturalcycles/time-lib'
|
|
3
10
|
import got from 'got'
|
|
4
11
|
import { inspectAny, InspectAnyOptions } from '..'
|
|
@@ -126,13 +133,42 @@ export class SlackService<CTX = any> {
|
|
|
126
133
|
})
|
|
127
134
|
}
|
|
128
135
|
|
|
129
|
-
kvToFields(kv:
|
|
136
|
+
kvToFields(kv: AnyObject): SlackAttachmentField[] {
|
|
130
137
|
return Object.entries(kv).map(([k, v]) => ({
|
|
131
138
|
title: k,
|
|
132
139
|
value: String(v),
|
|
133
140
|
short: String(v).length < 80,
|
|
134
141
|
}))
|
|
135
142
|
}
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* Returns a CommonLogger implementation based on this SlackService instance.
|
|
146
|
+
*/
|
|
147
|
+
getCommonLogger(opt: {
|
|
148
|
+
minLogLevel: CommonLogLevel
|
|
149
|
+
logChannel?: string
|
|
150
|
+
warnChannel?: string
|
|
151
|
+
errorChannel?: string
|
|
152
|
+
}): CommonLogger {
|
|
153
|
+
const { minLogLevel = 'log', logChannel, warnChannel, errorChannel } = opt
|
|
154
|
+
const defaultChannel = this.cfg.defaults?.channel || DEFAULTS.channel!
|
|
155
|
+
|
|
156
|
+
const q = new PQueue({
|
|
157
|
+
concurrency: 1,
|
|
158
|
+
})
|
|
159
|
+
|
|
160
|
+
return commonLoggerMinLevel(
|
|
161
|
+
{
|
|
162
|
+
log: (...args) =>
|
|
163
|
+
q.push(() => this.send({ items: args, channel: logChannel || defaultChannel })),
|
|
164
|
+
warn: (...args) =>
|
|
165
|
+
q.push(() => this.send({ items: args, channel: warnChannel || defaultChannel })),
|
|
166
|
+
error: (...args) =>
|
|
167
|
+
q.push(() => this.send({ items: args, channel: errorChannel || defaultChannel })),
|
|
168
|
+
},
|
|
169
|
+
minLogLevel,
|
|
170
|
+
)
|
|
171
|
+
}
|
|
136
172
|
}
|
|
137
173
|
|
|
138
174
|
export function slackDefaultMessagePrefixHook(msg: SlackMessage): string[] {
|
|
@@ -21,6 +21,11 @@ export interface TransformToNDJsonOptions {
|
|
|
21
21
|
* @default `\n`
|
|
22
22
|
*/
|
|
23
23
|
separator?: string
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* @experimental
|
|
27
|
+
*/
|
|
28
|
+
useFlatstr?: boolean
|
|
24
29
|
}
|
|
25
30
|
|
|
26
31
|
/**
|
|
@@ -29,7 +34,7 @@ export interface TransformToNDJsonOptions {
|
|
|
29
34
|
export function transformToNDJson<IN = any>(
|
|
30
35
|
opt: TransformToNDJsonOptions = {},
|
|
31
36
|
): TransformTyped<IN, string> {
|
|
32
|
-
const { strict = true, separator = '\n', sortObjects = false } = opt
|
|
37
|
+
const { strict = true, separator = '\n', sortObjects = false, useFlatstr = false } = opt
|
|
33
38
|
|
|
34
39
|
return new Transform({
|
|
35
40
|
objectMode: true,
|
|
@@ -39,7 +44,12 @@ export function transformToNDJson<IN = any>(
|
|
|
39
44
|
if (sortObjects) {
|
|
40
45
|
chunk = _sortObjectDeep(chunk as any)
|
|
41
46
|
}
|
|
42
|
-
|
|
47
|
+
|
|
48
|
+
if (useFlatstr) {
|
|
49
|
+
cb(null, flatstr(JSON.stringify(chunk) + separator))
|
|
50
|
+
} else {
|
|
51
|
+
cb(null, JSON.stringify(chunk) + separator)
|
|
52
|
+
}
|
|
43
53
|
} catch (err) {
|
|
44
54
|
console.error(err)
|
|
45
55
|
|
|
@@ -52,3 +62,12 @@ export function transformToNDJson<IN = any>(
|
|
|
52
62
|
},
|
|
53
63
|
})
|
|
54
64
|
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Based on: https://github.com/davidmarkclements/flatstr/blob/master/index.js
|
|
68
|
+
*/
|
|
69
|
+
function flatstr(s: any): string {
|
|
70
|
+
// eslint-disable-next-line
|
|
71
|
+
s | 0
|
|
72
|
+
return s
|
|
73
|
+
}
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
import { Transform } from 'stream'
|
|
2
|
+
import {
|
|
3
|
+
AggregatedError,
|
|
4
|
+
AsyncMapper,
|
|
5
|
+
CommonLogger,
|
|
6
|
+
ErrorMode,
|
|
7
|
+
pFilter,
|
|
8
|
+
PQueue,
|
|
9
|
+
} from '@naturalcycles/js-lib'
|
|
10
|
+
import { yellow } from '../../../colors'
|
|
11
|
+
import { TransformTyped } from '../../stream.model'
|
|
12
|
+
import { TransformMapOptions } from '../transformMap'
|
|
13
|
+
|
|
14
|
+
export function notNullishPredicate(item: any): boolean {
|
|
15
|
+
return item !== undefined && item !== null
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Like pMap, but for streams.
|
|
20
|
+
* Inspired by `through2`.
|
|
21
|
+
* Main feature is concurrency control (implemented via `through2-concurrent`) and convenient options.
|
|
22
|
+
* Using this allows native stream .pipe() to work and use backpressure.
|
|
23
|
+
*
|
|
24
|
+
* Only works in objectMode (due to through2Concurrent).
|
|
25
|
+
*
|
|
26
|
+
* Concurrency defaults to 16.
|
|
27
|
+
*
|
|
28
|
+
* If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
|
|
29
|
+
*/
|
|
30
|
+
export function transformMap2<IN = any, OUT = IN>(
|
|
31
|
+
mapper: AsyncMapper<IN, OUT>,
|
|
32
|
+
opt: TransformMapOptions<IN, OUT> = {},
|
|
33
|
+
): TransformTyped<IN, OUT> {
|
|
34
|
+
const {
|
|
35
|
+
concurrency = 16,
|
|
36
|
+
predicate = notNullishPredicate,
|
|
37
|
+
errorMode = ErrorMode.THROW_IMMEDIATELY,
|
|
38
|
+
flattenArrayOutput,
|
|
39
|
+
onError,
|
|
40
|
+
beforeFinal,
|
|
41
|
+
metric = 'stream',
|
|
42
|
+
logger = console,
|
|
43
|
+
} = opt
|
|
44
|
+
|
|
45
|
+
let index = -1
|
|
46
|
+
let isRejected = false
|
|
47
|
+
let errors = 0
|
|
48
|
+
const collectedErrors: Error[] = [] // only used if errorMode == THROW_AGGREGATED
|
|
49
|
+
|
|
50
|
+
const q = new PQueue({
|
|
51
|
+
concurrency,
|
|
52
|
+
resolveOn: 'start',
|
|
53
|
+
// debug: true,
|
|
54
|
+
})
|
|
55
|
+
|
|
56
|
+
return new Transform({
|
|
57
|
+
objectMode: true,
|
|
58
|
+
|
|
59
|
+
async final(cb) {
|
|
60
|
+
// console.log('transformMap final', {index}, q.inFlight, q.queueSize)
|
|
61
|
+
|
|
62
|
+
// wait for the current inFlight jobs to complete and push their results
|
|
63
|
+
await q.onIdle()
|
|
64
|
+
|
|
65
|
+
logErrorStats(logger, true)
|
|
66
|
+
|
|
67
|
+
await beforeFinal?.() // call beforeFinal if defined
|
|
68
|
+
|
|
69
|
+
if (collectedErrors.length) {
|
|
70
|
+
// emit Aggregated error
|
|
71
|
+
cb(new AggregatedError(collectedErrors))
|
|
72
|
+
} else {
|
|
73
|
+
// emit no error
|
|
74
|
+
cb()
|
|
75
|
+
}
|
|
76
|
+
},
|
|
77
|
+
|
|
78
|
+
async transform(this: Transform, chunk: IN, _encoding, cb) {
|
|
79
|
+
index++
|
|
80
|
+
// console.log('transform', {index})
|
|
81
|
+
|
|
82
|
+
// Stop processing if THROW_IMMEDIATELY mode is used
|
|
83
|
+
if (isRejected && errorMode === ErrorMode.THROW_IMMEDIATELY) return cb()
|
|
84
|
+
|
|
85
|
+
// It resolves when it is successfully STARTED execution.
|
|
86
|
+
// If it's queued instead - it'll wait and resolve only upon START.
|
|
87
|
+
await q.push(async () => {
|
|
88
|
+
try {
|
|
89
|
+
const currentIndex = index // because we need to pass it to 2 functions - mapper and predicate. Refers to INPUT index (since it may return multiple outputs)
|
|
90
|
+
const res = await mapper(chunk, currentIndex)
|
|
91
|
+
const passedResults = await pFilter(
|
|
92
|
+
flattenArrayOutput && Array.isArray(res) ? res : [res],
|
|
93
|
+
async r => await predicate(r, currentIndex),
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
passedResults.forEach(r => this.push(r))
|
|
97
|
+
} catch (err) {
|
|
98
|
+
logger.error(err)
|
|
99
|
+
|
|
100
|
+
errors++
|
|
101
|
+
|
|
102
|
+
logErrorStats(logger)
|
|
103
|
+
|
|
104
|
+
if (onError) {
|
|
105
|
+
try {
|
|
106
|
+
onError(err, chunk)
|
|
107
|
+
} catch {}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
if (errorMode === ErrorMode.THROW_IMMEDIATELY) {
|
|
111
|
+
isRejected = true
|
|
112
|
+
// Emit error immediately
|
|
113
|
+
// return cb(err as Error)
|
|
114
|
+
return this.emit('error', err as Error)
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
if (errorMode === ErrorMode.THROW_AGGREGATED) {
|
|
118
|
+
collectedErrors.push(err as Error)
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
})
|
|
122
|
+
|
|
123
|
+
// Resolved, which means it STARTED processing
|
|
124
|
+
// This means we can take more load
|
|
125
|
+
cb()
|
|
126
|
+
},
|
|
127
|
+
})
|
|
128
|
+
|
|
129
|
+
function logErrorStats(logger: CommonLogger, final = false): void {
|
|
130
|
+
if (!errors) return
|
|
131
|
+
|
|
132
|
+
logger.log(`${metric} ${final ? 'final ' : ''}errors: ${yellow(errors)}`)
|
|
133
|
+
}
|
|
134
|
+
}
|
|
@@ -3,6 +3,7 @@ import {
|
|
|
3
3
|
AggregatedError,
|
|
4
4
|
AsyncMapper,
|
|
5
5
|
AsyncPredicate,
|
|
6
|
+
CommonLogger,
|
|
6
7
|
ErrorMode,
|
|
7
8
|
pFilter,
|
|
8
9
|
} from '@naturalcycles/js-lib'
|
|
@@ -57,6 +58,8 @@ export interface TransformMapOptions<IN = any, OUT = IN> {
|
|
|
57
58
|
* If defined - called BEFORE `final()` callback is called.
|
|
58
59
|
*/
|
|
59
60
|
beforeFinal?: () => any
|
|
61
|
+
|
|
62
|
+
logger?: CommonLogger
|
|
60
63
|
}
|
|
61
64
|
|
|
62
65
|
export function notNullishPredicate(item: any): boolean {
|
|
@@ -87,6 +90,7 @@ export function transformMap<IN = any, OUT = IN>(
|
|
|
87
90
|
onError,
|
|
88
91
|
beforeFinal,
|
|
89
92
|
metric = 'stream',
|
|
93
|
+
logger = console,
|
|
90
94
|
} = opt
|
|
91
95
|
|
|
92
96
|
let index = -1
|
|
@@ -101,7 +105,7 @@ export function transformMap<IN = any, OUT = IN>(
|
|
|
101
105
|
async final(cb) {
|
|
102
106
|
// console.log('transformMap final')
|
|
103
107
|
|
|
104
|
-
logErrorStats(true)
|
|
108
|
+
logErrorStats(logger, true)
|
|
105
109
|
|
|
106
110
|
await beforeFinal?.() // call beforeFinal if defined
|
|
107
111
|
|
|
@@ -144,11 +148,11 @@ export function transformMap<IN = any, OUT = IN>(
|
|
|
144
148
|
cb() // done processing
|
|
145
149
|
}
|
|
146
150
|
} catch (err) {
|
|
147
|
-
|
|
151
|
+
logger.error(err)
|
|
148
152
|
|
|
149
153
|
errors++
|
|
150
154
|
|
|
151
|
-
logErrorStats()
|
|
155
|
+
logErrorStats(logger)
|
|
152
156
|
|
|
153
157
|
if (onError) {
|
|
154
158
|
try {
|
|
@@ -172,9 +176,9 @@ export function transformMap<IN = any, OUT = IN>(
|
|
|
172
176
|
},
|
|
173
177
|
)
|
|
174
178
|
|
|
175
|
-
function logErrorStats(final = false): void {
|
|
179
|
+
function logErrorStats(logger: CommonLogger, final = false): void {
|
|
176
180
|
if (!errors) return
|
|
177
181
|
|
|
178
|
-
|
|
182
|
+
logger.log(`${metric} ${final ? 'final ' : ''}errors: ${yellow(errors)}`)
|
|
179
183
|
}
|
|
180
184
|
}
|
package/src/string/inspectAny.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { inspect, InspectOptions } from 'util'
|
|
2
|
-
import { StringifyAnyOptions, _stringifyAny } from '@naturalcycles/js-lib'
|
|
2
|
+
import { StringifyAnyOptions, _stringifyAny, JsonStringifyFunction } from '@naturalcycles/js-lib'
|
|
3
3
|
|
|
4
4
|
export interface InspectAnyOptions extends StringifyAnyOptions, InspectOptions {}
|
|
5
5
|
|
|
@@ -8,6 +8,11 @@ const INSPECT_OPT: InspectOptions = {
|
|
|
8
8
|
depth: 10, // default: 2
|
|
9
9
|
}
|
|
10
10
|
|
|
11
|
+
/**
|
|
12
|
+
* Just a convenience export of a const that fulfills the JsonStringifyFunction interface.
|
|
13
|
+
*/
|
|
14
|
+
export const inspectAnyStringifyFn: JsonStringifyFunction = obj => inspectAny(obj)
|
|
15
|
+
|
|
11
16
|
/**
|
|
12
17
|
* Transforms ANY to human-readable string (via util.inspect mainly).
|
|
13
18
|
* Safe (no error throwing).
|
|
@@ -6,6 +6,7 @@ import {
|
|
|
6
6
|
_filterNullishValues,
|
|
7
7
|
_isObject,
|
|
8
8
|
_substringBefore,
|
|
9
|
+
CommonLogger,
|
|
9
10
|
} from '@naturalcycles/js-lib'
|
|
10
11
|
import Ajv, { ValidateFunction } from 'ajv'
|
|
11
12
|
import { inspectAny, requireFileToExist } from '../../index'
|
|
@@ -56,6 +57,11 @@ export interface AjvSchemaCfg {
|
|
|
56
57
|
*/
|
|
57
58
|
logErrors: boolean
|
|
58
59
|
|
|
60
|
+
/**
|
|
61
|
+
* Default to `console`
|
|
62
|
+
*/
|
|
63
|
+
logger: CommonLogger
|
|
64
|
+
|
|
59
65
|
/**
|
|
60
66
|
* Option of Ajv.
|
|
61
67
|
* If set to true - will mutate your input objects!
|
|
@@ -76,6 +82,7 @@ export class AjvSchema<T = unknown> {
|
|
|
76
82
|
private constructor(public schema: JsonSchema<T>, cfg: Partial<AjvSchemaCfg> = {}) {
|
|
77
83
|
this.cfg = {
|
|
78
84
|
logErrors: true,
|
|
85
|
+
logger: console,
|
|
79
86
|
separator: '\n',
|
|
80
87
|
...cfg,
|
|
81
88
|
ajv:
|
|
@@ -168,7 +175,7 @@ export class AjvSchema<T = unknown> {
|
|
|
168
175
|
message = [message, 'Input: ' + strValue].join(separator)
|
|
169
176
|
|
|
170
177
|
if (logErrors) {
|
|
171
|
-
|
|
178
|
+
this.cfg.logger.error(errors)
|
|
172
179
|
}
|
|
173
180
|
|
|
174
181
|
return new AjvValidationError(
|