@naturalcycles/nodejs-lib 12.49.0 → 12.53.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/got/getGot.js +4 -3
- package/dist/got/got.model.d.ts +5 -1
- package/dist/index.d.ts +3 -3
- package/dist/index.js +3 -3
- package/dist/log/debug.d.ts +0 -10
- package/dist/log/debug.js +1 -18
- package/dist/security/secret.util.js +3 -5
- package/dist/slack/slack.service.d.ts +1 -1
- package/dist/slack/slack.service.js +20 -28
- package/dist/slack/slack.service.model.d.ts +16 -21
- package/dist/stream/ndjson/transformToNDJson.d.ts +4 -0
- package/dist/stream/ndjson/transformToNDJson.js +15 -2
- package/dist/stream/transform/transformLogProgress.js +1 -1
- package/dist/string/inspectAny.d.ts +5 -1
- package/dist/string/inspectAny.js +6 -1
- package/dist/validation/ajv/ajvValidationError.js +0 -14
- package/dist/validation/joi/joi.validation.error.js +0 -14
- package/dist/validation/joi/joi.validation.util.js +1 -0
- package/package.json +1 -1
- package/src/got/getGot.ts +6 -4
- package/src/got/got.model.ts +6 -1
- package/src/index.ts +3 -3
- package/src/log/debug.ts +1 -25
- package/src/security/secret.util.ts +4 -7
- package/src/slack/slack.service.model.ts +18 -25
- package/src/slack/slack.service.ts +28 -41
- package/src/stream/ndjson/transformToNDJson.ts +21 -2
- package/src/stream/transform/transformLogProgress.ts +1 -2
- package/src/string/inspectAny.ts +6 -1
- package/src/validation/ajv/ajvValidationError.ts +0 -15
- package/src/validation/joi/joi.validation.error.ts +0 -15
- package/src/validation/joi/joi.validation.util.ts +1 -0
package/dist/got/getGot.js
CHANGED
|
@@ -13,6 +13,7 @@ const colors_1 = require("../colors");
|
|
|
13
13
|
* 3. Reasonable defaults(tm), e.g non-infinite Timeout
|
|
14
14
|
*/
|
|
15
15
|
function getGot(opt = {}) {
|
|
16
|
+
opt.logger || (opt.logger = console);
|
|
16
17
|
return got_1.default.extend({
|
|
17
18
|
// Most-important is to set to anything non-empty (so, requests don't "hang" by default).
|
|
18
19
|
// Should be long enough to handle for slow responses from scaled cloud APIs in times of spikes
|
|
@@ -93,7 +94,7 @@ function gotBeforeRequestHook(opt) {
|
|
|
93
94
|
};
|
|
94
95
|
if (opt.logStart) {
|
|
95
96
|
const shortUrl = getShortUrl(opt, options.url, options.prefixUrl);
|
|
96
|
-
|
|
97
|
+
opt.logger.log([(0, colors_1.dimGrey)(' >>'), (0, colors_1.dimGrey)(options.method), (0, colors_1.grey)(shortUrl)].join(' '));
|
|
97
98
|
}
|
|
98
99
|
};
|
|
99
100
|
}
|
|
@@ -104,7 +105,7 @@ function gotAfterResponseHook(opt = {}) {
|
|
|
104
105
|
const { started } = resp.request.options.context;
|
|
105
106
|
const { url, prefixUrl, method } = resp.request.options;
|
|
106
107
|
const shortUrl = getShortUrl(opt, url, prefixUrl);
|
|
107
|
-
|
|
108
|
+
opt.logger.log([
|
|
108
109
|
(0, colors_1.dimGrey)(' <<'),
|
|
109
110
|
coloredHttpCode(resp.statusCode),
|
|
110
111
|
(0, colors_1.dimGrey)(method),
|
|
@@ -117,7 +118,7 @@ function gotAfterResponseHook(opt = {}) {
|
|
|
117
118
|
}
|
|
118
119
|
// Error responses are not logged, cause they're included in Error message already
|
|
119
120
|
if (opt.logResponse && success) {
|
|
120
|
-
|
|
121
|
+
opt.logger.log((0, __1.inspectAny)(resp.body, { maxLen: opt.maxResponseLength }));
|
|
121
122
|
}
|
|
122
123
|
return resp;
|
|
123
124
|
};
|
package/dist/got/got.model.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { AnyObject } from '@naturalcycles/js-lib';
|
|
1
|
+
import { AnyObject, CommonLogger } from '@naturalcycles/js-lib';
|
|
2
2
|
import type { Options } from 'got';
|
|
3
3
|
export interface GetGotOptions extends Options {
|
|
4
4
|
/**
|
|
@@ -27,6 +27,10 @@ export interface GetGotOptions extends Options {
|
|
|
27
27
|
* Set to false to strip searchParams from url when logging (both success and error)
|
|
28
28
|
*/
|
|
29
29
|
logWithSearchParams?: boolean;
|
|
30
|
+
/**
|
|
31
|
+
* Defaults to `console`
|
|
32
|
+
*/
|
|
33
|
+
logger?: CommonLogger;
|
|
30
34
|
/**
|
|
31
35
|
* Max length of response object before it's truncated.
|
|
32
36
|
*
|
package/dist/index.d.ts
CHANGED
|
@@ -7,7 +7,7 @@ import { tableDiff, TableDiffOptions } from './diff/tableDiff';
|
|
|
7
7
|
import { getGot } from './got/getGot';
|
|
8
8
|
import { GetGotOptions } from './got/got.model';
|
|
9
9
|
import { memoryUsage, memoryUsageFull, processSharedUtil } from './infra/process.util';
|
|
10
|
-
import { Debug,
|
|
10
|
+
import { Debug, IDebug, IDebugger } from './log/debug';
|
|
11
11
|
import { base64ToBuffer, base64ToString, bufferToBase64, hash, md5, hashAsBuffer, md5AsBuffer, stringToBase64 } from './security/hash.util';
|
|
12
12
|
import { ALPHABET_ALPHANUMERIC, ALPHABET_ALPHANUMERIC_LOWERCASE, ALPHABET_ALPHANUMERIC_UPPERCASE, ALPHABET_LOWERCASE, ALPHABET_NUMBER, ALPHABET_UPPERCASE, stringId, stringIdAsync, stringIdUnsafe } from './security/id.util';
|
|
13
13
|
import { getSecretMap, loadSecretsFromEnv, loadSecretsFromJsonFile, removeSecretsFromEnv, secret, secretOptional, setSecretMap } from './security/secret.util';
|
|
@@ -51,7 +51,7 @@ import { writableForEach } from './stream/writable/writableForEach';
|
|
|
51
51
|
import { writableFork } from './stream/writable/writableFork';
|
|
52
52
|
import { writablePushToArray } from './stream/writable/writablePushToArray';
|
|
53
53
|
import { writableVoid } from './stream/writable/writableVoid';
|
|
54
|
-
import { inspectAny, InspectAnyOptions } from './string/inspectAny';
|
|
54
|
+
import { inspectAny, InspectAnyOptions, inspectAnyStringifyFn } from './string/inspectAny';
|
|
55
55
|
import { requireEnvKeys, requireFileToExist } from './util/env.util';
|
|
56
56
|
import { LRUMemoCache } from './util/lruMemoCache';
|
|
57
57
|
import { gunzipBuffer, gunzipToString, gzipBuffer, gzipString, unzipBuffer, unzipToString, zipBuffer, zipString } from './util/zip.util';
|
|
@@ -66,4 +66,4 @@ import { JoiValidationError, JoiValidationErrorData } from './validation/joi/joi
|
|
|
66
66
|
import { convert, getValidationResult, isValid, JoiValidationResult, undefinedIfInvalid, validate } from './validation/joi/joi.validation.util';
|
|
67
67
|
import { sanitizeHTML, SanitizeHTMLOptions } from './validation/sanitize.util';
|
|
68
68
|
export type { JoiValidationErrorData, JoiValidationResult, ValidationErrorItem, ExtendedJoi, SchemaTyped, AnySchema, AnySchemaTyped, ArraySchemaTyped, BooleanSchemaTyped, NumberSchemaTyped, ObjectSchemaTyped, StringSchemaTyped, IDebug, IDebugger, SlackServiceCfg, SlackMessage, SlackMessageProps, SlackApiBody, SlackMessagePrefixHook, ReadableTyped, WritableTyped, TransformTyped, PipelineFromNDJsonFileOptions, PipelineToNDJsonFileOptions, TransformJsonParseOptions, TransformToNDJsonOptions, TransformMapOptions, TransformMapSyncOptions, NDJSONStreamForEachOptions, TransformOptions, TransformLogProgressOptions, TransformMultiThreadedOptions, WorkerClassInterface, WorkerInput, WorkerOutput, TableDiffOptions, InspectAnyOptions, Got, GetGotOptions, AfterResponseHook, BeforeErrorHook, BeforeRequestHook, AjvValidationOptions, AjvSchemaCfg, AjvValidationErrorData, SanitizeHTMLOptions, };
|
|
69
|
-
export { JoiValidationError, validate, getValidationResult, isValid, undefinedIfInvalid, convert, Joi, booleanSchema, booleanDefaultToFalseSchema, stringSchema, numberSchema, integerSchema, percentageSchema, dateStringSchema, arraySchema, binarySchema, objectSchema, oneOfSchema, anySchema, anyObjectSchema, baseDBEntitySchema, savedDBEntitySchema, idSchema, unixTimestampSchema, verSchema, emailSchema, SEM_VER_PATTERN, semVerSchema, userAgentSchema, utcOffsetSchema, ipAddressSchema, slugSchema, urlSchema, processSharedUtil, zipBuffer, gzipBuffer, unzipBuffer, gunzipBuffer, zipString, gzipString, unzipToString, gunzipToString, requireEnvKeys, requireFileToExist, LRUMemoCache, stringId, stringIdAsync, stringIdUnsafe, ALPHABET_NUMBER, ALPHABET_LOWERCASE, ALPHABET_UPPERCASE, ALPHABET_ALPHANUMERIC_LOWERCASE, ALPHABET_ALPHANUMERIC_UPPERCASE, ALPHABET_ALPHANUMERIC, md5, hash, hashAsBuffer, md5AsBuffer, stringToBase64, base64ToString, bufferToBase64, base64ToBuffer, Debug,
|
|
69
|
+
export { JoiValidationError, validate, getValidationResult, isValid, undefinedIfInvalid, convert, Joi, booleanSchema, booleanDefaultToFalseSchema, stringSchema, numberSchema, integerSchema, percentageSchema, dateStringSchema, arraySchema, binarySchema, objectSchema, oneOfSchema, anySchema, anyObjectSchema, baseDBEntitySchema, savedDBEntitySchema, idSchema, unixTimestampSchema, verSchema, emailSchema, SEM_VER_PATTERN, semVerSchema, userAgentSchema, utcOffsetSchema, ipAddressSchema, slugSchema, urlSchema, processSharedUtil, zipBuffer, gzipBuffer, unzipBuffer, gunzipBuffer, zipString, gzipString, unzipToString, gunzipToString, requireEnvKeys, requireFileToExist, LRUMemoCache, stringId, stringIdAsync, stringIdUnsafe, ALPHABET_NUMBER, ALPHABET_LOWERCASE, ALPHABET_UPPERCASE, ALPHABET_ALPHANUMERIC_LOWERCASE, ALPHABET_ALPHANUMERIC_UPPERCASE, ALPHABET_ALPHANUMERIC, md5, hash, hashAsBuffer, md5AsBuffer, stringToBase64, base64ToString, bufferToBase64, base64ToBuffer, Debug, getSecretMap, setSecretMap, loadSecretsFromEnv, loadSecretsFromJsonFile, removeSecretsFromEnv, secret, secretOptional, memoryUsage, memoryUsageFull, SlackService, slackDefaultMessagePrefixHook, readableCreate, readableFrom, readableFromArray, readableToArray, readableForEach, readableForEachSync, readableMap, readableMapToArray, _pipeline, transformBuffer, ndjsonMap, ndJsonFileRead, ndJsonFileWrite, ndjsonStreamForEach, pipelineFromNDJsonFile, pipelineToNDJsonFile, NDJsonStats, streamToNDJsonFile, transformJsonParse, bufferReviver, transformToNDJson, transformFilter, transformFilterSync, transformMap, transformMapSync, transformMapSimple, transformNoOp, writableForEach, writablePushToArray, transformSplit, transformToString, transformToArray, transformTap, transformLogProgress, transformLimit, writableVoid, writableFork, transformMultiThreaded, BaseWorkerClass, tableDiff, inspectAny, inspectAnyStringifyFn, getGot, HTTPError, TimeoutError, _chunkBuffer, Ajv, getAjv, AjvSchema, AjvValidationError, readJsonSchemas, readAjvSchemas, hasColors, sanitizeHTML, };
|
package/dist/index.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.ALPHABET_LOWERCASE = exports.ALPHABET_NUMBER = exports.stringIdUnsafe = exports.stringIdAsync = exports.stringId = exports.LRUMemoCache = exports.requireFileToExist = exports.requireEnvKeys = exports.gunzipToString = exports.unzipToString = exports.gzipString = exports.zipString = exports.gunzipBuffer = exports.unzipBuffer = exports.gzipBuffer = exports.zipBuffer = exports.processSharedUtil = exports.urlSchema = exports.slugSchema = exports.ipAddressSchema = exports.utcOffsetSchema = exports.userAgentSchema = exports.semVerSchema = exports.SEM_VER_PATTERN = exports.emailSchema = exports.verSchema = exports.unixTimestampSchema = exports.idSchema = exports.savedDBEntitySchema = exports.baseDBEntitySchema = exports.anyObjectSchema = exports.anySchema = exports.oneOfSchema = exports.objectSchema = exports.binarySchema = exports.arraySchema = exports.dateStringSchema = exports.percentageSchema = exports.integerSchema = exports.numberSchema = exports.stringSchema = exports.booleanDefaultToFalseSchema = exports.booleanSchema = exports.Joi = exports.convert = exports.undefinedIfInvalid = exports.isValid = exports.getValidationResult = exports.validate = exports.JoiValidationError = void 0;
|
|
4
|
-
exports.transformMapSync = exports.transformMap = exports.transformFilterSync = exports.transformFilter = exports.transformToNDJson = exports.bufferReviver = exports.transformJsonParse = exports.streamToNDJsonFile = exports.NDJsonStats = exports.pipelineToNDJsonFile = exports.pipelineFromNDJsonFile = exports.ndjsonStreamForEach = exports.ndJsonFileWrite = exports.ndJsonFileRead = exports.ndjsonMap = exports.transformBuffer = exports._pipeline = exports.readableMapToArray = exports.readableMap = exports.readableForEachSync = exports.readableForEach = exports.readableToArray = exports.readableFromArray = exports.readableFrom = exports.readableCreate = exports.slackDefaultMessagePrefixHook = exports.SlackService = exports.memoryUsageFull = exports.memoryUsage = exports.secretOptional = exports.secret = exports.removeSecretsFromEnv = exports.loadSecretsFromJsonFile = exports.loadSecretsFromEnv = exports.setSecretMap = exports.getSecretMap = exports.
|
|
5
|
-
exports.sanitizeHTML = exports.hasColors = exports.readAjvSchemas = exports.readJsonSchemas = exports.AjvValidationError = exports.AjvSchema = exports.getAjv = exports.Ajv = exports._chunkBuffer = exports.TimeoutError = exports.HTTPError = exports.getGot = exports.inspectAny = exports.tableDiff = exports.BaseWorkerClass = exports.transformMultiThreaded = exports.writableFork = exports.writableVoid = exports.transformLimit = exports.transformLogProgress = exports.transformTap = exports.transformToArray = exports.transformToString = exports.transformSplit = exports.writablePushToArray = exports.writableForEach = exports.transformNoOp =
|
|
4
|
+
exports.transformMapSimple = exports.transformMapSync = exports.transformMap = exports.transformFilterSync = exports.transformFilter = exports.transformToNDJson = exports.bufferReviver = exports.transformJsonParse = exports.streamToNDJsonFile = exports.NDJsonStats = exports.pipelineToNDJsonFile = exports.pipelineFromNDJsonFile = exports.ndjsonStreamForEach = exports.ndJsonFileWrite = exports.ndJsonFileRead = exports.ndjsonMap = exports.transformBuffer = exports._pipeline = exports.readableMapToArray = exports.readableMap = exports.readableForEachSync = exports.readableForEach = exports.readableToArray = exports.readableFromArray = exports.readableFrom = exports.readableCreate = exports.slackDefaultMessagePrefixHook = exports.SlackService = exports.memoryUsageFull = exports.memoryUsage = exports.secretOptional = exports.secret = exports.removeSecretsFromEnv = exports.loadSecretsFromJsonFile = exports.loadSecretsFromEnv = exports.setSecretMap = exports.getSecretMap = exports.Debug = exports.base64ToBuffer = exports.bufferToBase64 = exports.base64ToString = exports.stringToBase64 = exports.md5AsBuffer = exports.hashAsBuffer = exports.hash = exports.md5 = exports.ALPHABET_ALPHANUMERIC = exports.ALPHABET_ALPHANUMERIC_UPPERCASE = exports.ALPHABET_ALPHANUMERIC_LOWERCASE = exports.ALPHABET_UPPERCASE = void 0;
|
|
5
|
+
exports.sanitizeHTML = exports.hasColors = exports.readAjvSchemas = exports.readJsonSchemas = exports.AjvValidationError = exports.AjvSchema = exports.getAjv = exports.Ajv = exports._chunkBuffer = exports.TimeoutError = exports.HTTPError = exports.getGot = exports.inspectAnyStringifyFn = exports.inspectAny = exports.tableDiff = exports.BaseWorkerClass = exports.transformMultiThreaded = exports.writableFork = exports.writableVoid = exports.transformLimit = exports.transformLogProgress = exports.transformTap = exports.transformToArray = exports.transformToString = exports.transformSplit = exports.writablePushToArray = exports.writableForEach = exports.transformNoOp = void 0;
|
|
6
6
|
const ajv_1 = require("ajv");
|
|
7
7
|
exports.Ajv = ajv_1.default;
|
|
8
8
|
const got_1 = require("got");
|
|
@@ -20,7 +20,6 @@ Object.defineProperty(exports, "memoryUsageFull", { enumerable: true, get: funct
|
|
|
20
20
|
Object.defineProperty(exports, "processSharedUtil", { enumerable: true, get: function () { return process_util_1.processSharedUtil; } });
|
|
21
21
|
const debug_1 = require("./log/debug");
|
|
22
22
|
Object.defineProperty(exports, "Debug", { enumerable: true, get: function () { return debug_1.Debug; } });
|
|
23
|
-
Object.defineProperty(exports, "DebugLogLevel", { enumerable: true, get: function () { return debug_1.DebugLogLevel; } });
|
|
24
23
|
const hash_util_1 = require("./security/hash.util");
|
|
25
24
|
Object.defineProperty(exports, "base64ToBuffer", { enumerable: true, get: function () { return hash_util_1.base64ToBuffer; } });
|
|
26
25
|
Object.defineProperty(exports, "base64ToString", { enumerable: true, get: function () { return hash_util_1.base64ToString; } });
|
|
@@ -129,6 +128,7 @@ const writableVoid_1 = require("./stream/writable/writableVoid");
|
|
|
129
128
|
Object.defineProperty(exports, "writableVoid", { enumerable: true, get: function () { return writableVoid_1.writableVoid; } });
|
|
130
129
|
const inspectAny_1 = require("./string/inspectAny");
|
|
131
130
|
Object.defineProperty(exports, "inspectAny", { enumerable: true, get: function () { return inspectAny_1.inspectAny; } });
|
|
131
|
+
Object.defineProperty(exports, "inspectAnyStringifyFn", { enumerable: true, get: function () { return inspectAny_1.inspectAnyStringifyFn; } });
|
|
132
132
|
const env_util_1 = require("./util/env.util");
|
|
133
133
|
Object.defineProperty(exports, "requireEnvKeys", { enumerable: true, get: function () { return env_util_1.requireEnvKeys; } });
|
|
134
134
|
Object.defineProperty(exports, "requireFileToExist", { enumerable: true, get: function () { return env_util_1.requireFileToExist; } });
|
package/dist/log/debug.d.ts
CHANGED
|
@@ -14,20 +14,10 @@ export interface DebugFormatters {
|
|
|
14
14
|
}
|
|
15
15
|
export interface IDebugger {
|
|
16
16
|
(...args: any[]): void;
|
|
17
|
-
debug: (...args: any[]) => void;
|
|
18
|
-
info: (...args: any[]) => void;
|
|
19
|
-
warn: (...args: any[]) => void;
|
|
20
|
-
error: (...args: any[]) => void;
|
|
21
17
|
color: string;
|
|
22
18
|
enabled: boolean;
|
|
23
19
|
log: (...args: any[]) => any;
|
|
24
20
|
namespace: string;
|
|
25
21
|
destroy: () => boolean;
|
|
26
22
|
}
|
|
27
|
-
export declare enum DebugLogLevel {
|
|
28
|
-
debug = "debug",
|
|
29
|
-
info = "info",
|
|
30
|
-
warn = "warn",
|
|
31
|
-
error = "error"
|
|
32
|
-
}
|
|
33
23
|
export declare const Debug: IDebug;
|
package/dist/log/debug.js
CHANGED
|
@@ -1,28 +1,11 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.Debug =
|
|
4
|
-
var DebugLogLevel;
|
|
5
|
-
(function (DebugLogLevel) {
|
|
6
|
-
DebugLogLevel["debug"] = "debug";
|
|
7
|
-
DebugLogLevel["info"] = "info";
|
|
8
|
-
DebugLogLevel["warn"] = "warn";
|
|
9
|
-
DebugLogLevel["error"] = "error";
|
|
10
|
-
})(DebugLogLevel = exports.DebugLogLevel || (exports.DebugLogLevel = {}));
|
|
3
|
+
exports.Debug = void 0;
|
|
11
4
|
const originalDebug = require('debug');
|
|
12
5
|
// eslint-disable-next-line @typescript-eslint/naming-convention
|
|
13
6
|
exports.Debug = ((namespace) => {
|
|
14
7
|
const instance = originalDebug(namespace);
|
|
15
8
|
instance.log = console.log.bind(console); // this enables colors for objects
|
|
16
|
-
instance.info = instance.bind(instance);
|
|
17
|
-
const instanceDebug = originalDebug([namespace, 'debug'].join(':'));
|
|
18
|
-
instanceDebug.log = console.debug.bind(console);
|
|
19
|
-
instance.debug = instanceDebug.bind(instanceDebug);
|
|
20
|
-
const instanceWarn = originalDebug([namespace, 'warn'].join(':'));
|
|
21
|
-
instanceWarn.log = console.warn.bind(console);
|
|
22
|
-
instance.warn = instanceWarn.bind(instanceWarn);
|
|
23
|
-
const instanceError = originalDebug([namespace, 'error'].join(':'));
|
|
24
|
-
instanceError.log = console.error.bind(console);
|
|
25
|
-
instance.error = instanceError.bind(instanceError);
|
|
26
9
|
return instance;
|
|
27
10
|
});
|
|
28
11
|
exports.Debug.coerce = originalDebug.coerce.bind(originalDebug);
|
|
@@ -5,8 +5,6 @@ const fs = require("fs");
|
|
|
5
5
|
const __1 = require("..");
|
|
6
6
|
const crypto_util_1 = require("./crypto.util");
|
|
7
7
|
let loaded = false;
|
|
8
|
-
// it's wrapped to be able to pipe console.* to Stackdriver
|
|
9
|
-
const getLog = () => (0, __1.Debug)('nc:nodejs-lib:secret');
|
|
10
8
|
const secretMap = {};
|
|
11
9
|
/**
|
|
12
10
|
* Loads plaintext secrets from process.env, removes them, stores locally.
|
|
@@ -25,7 +23,7 @@ function loadSecretsFromEnv() {
|
|
|
25
23
|
delete process.env[k];
|
|
26
24
|
});
|
|
27
25
|
loaded = true;
|
|
28
|
-
|
|
26
|
+
console.log(`${Object.keys(secrets).length} secret(s) loaded from process.env: ${Object.keys(secrets).join(', ')}`);
|
|
29
27
|
}
|
|
30
28
|
exports.loadSecretsFromEnv = loadSecretsFromEnv;
|
|
31
29
|
/**
|
|
@@ -58,7 +56,7 @@ function loadSecretsFromJsonFile(filePath, SECRET_ENCRYPTION_KEY) {
|
|
|
58
56
|
}
|
|
59
57
|
Object.entries(secrets).forEach(([k, v]) => (secretMap[k.toUpperCase()] = v));
|
|
60
58
|
loaded = true;
|
|
61
|
-
|
|
59
|
+
console.log(`${Object.keys(secrets).length} secret(s) loaded from ${filePath}: ${Object.keys(secrets)
|
|
62
60
|
.map(s => s.toUpperCase())
|
|
63
61
|
.join(', ')}`);
|
|
64
62
|
}
|
|
@@ -91,7 +89,7 @@ exports.getSecretMap = getSecretMap;
|
|
|
91
89
|
function setSecretMap(map) {
|
|
92
90
|
Object.keys(secretMap).forEach(k => delete secretMap[k]);
|
|
93
91
|
Object.entries(map).forEach(([k, v]) => (secretMap[k.toUpperCase()] = v));
|
|
94
|
-
|
|
92
|
+
console.log(`setSecretMap set ${Object.keys(secretMap).length} secret(s): ${Object.keys(map)
|
|
95
93
|
.map(s => s.toUpperCase())
|
|
96
94
|
.join(', ')}`);
|
|
97
95
|
}
|
|
@@ -19,7 +19,7 @@ export declare class SlackService<CTX = any> {
|
|
|
19
19
|
* Allows to "log" many things at once, similar to `console.log(one, two, three).
|
|
20
20
|
*/
|
|
21
21
|
log(...items: any[]): Promise<void>;
|
|
22
|
-
send(
|
|
22
|
+
send(input: SlackMessage<CTX> | string, ctx?: CTX): Promise<void>;
|
|
23
23
|
kvToFields(kv: StringMap<any>): SlackAttachmentField[];
|
|
24
24
|
}
|
|
25
25
|
export declare function slackDefaultMessagePrefixHook(msg: SlackMessage): string[];
|
|
@@ -1,22 +1,22 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.slackDefaultMessagePrefixHook = exports.SlackService = void 0;
|
|
4
|
+
const js_lib_1 = require("@naturalcycles/js-lib");
|
|
4
5
|
const time_lib_1 = require("@naturalcycles/time-lib");
|
|
5
6
|
const got_1 = require("got");
|
|
6
7
|
const __1 = require("..");
|
|
7
8
|
const GAE = !!process.env['GAE_INSTANCE'];
|
|
8
|
-
const DEFAULTS =
|
|
9
|
+
const DEFAULTS = {
|
|
9
10
|
username: 'bot',
|
|
10
11
|
channel: '#log',
|
|
11
12
|
icon_emoji: ':spider_web:',
|
|
12
13
|
items: 'no text',
|
|
13
|
-
}
|
|
14
|
+
};
|
|
14
15
|
const INSPECT_OPT = {
|
|
15
16
|
colors: false,
|
|
16
17
|
includeErrorData: true,
|
|
17
18
|
includeErrorStack: true,
|
|
18
19
|
};
|
|
19
|
-
const log = (0, __1.Debug)('nc:nodejs-lib:slack');
|
|
20
20
|
/**
|
|
21
21
|
* Has 2 main methods:
|
|
22
22
|
*
|
|
@@ -33,7 +33,12 @@ class SlackService {
|
|
|
33
33
|
constructor(cfg) {
|
|
34
34
|
this.cfg = {
|
|
35
35
|
messagePrefixHook: slackDefaultMessagePrefixHook,
|
|
36
|
+
logger: console,
|
|
36
37
|
...cfg,
|
|
38
|
+
inspectOptions: {
|
|
39
|
+
...INSPECT_OPT,
|
|
40
|
+
...cfg.inspectOptions,
|
|
41
|
+
},
|
|
37
42
|
};
|
|
38
43
|
}
|
|
39
44
|
/**
|
|
@@ -45,38 +50,29 @@ class SlackService {
|
|
|
45
50
|
items: items.length === 1 ? items[0] : items,
|
|
46
51
|
});
|
|
47
52
|
}
|
|
48
|
-
async send(
|
|
49
|
-
const { webhookUrl, messagePrefixHook } = this.cfg;
|
|
53
|
+
async send(input, ctx) {
|
|
54
|
+
const { webhookUrl, messagePrefixHook, inspectOptions } = this.cfg;
|
|
50
55
|
// If String is passed as first argument - just transform it to a full SlackMessage
|
|
51
|
-
|
|
52
|
-
msg = {
|
|
53
|
-
items: msg,
|
|
54
|
-
};
|
|
55
|
-
}
|
|
56
|
+
const msg = typeof input === 'string' ? { items: input } : input;
|
|
56
57
|
if (ctx !== undefined) {
|
|
57
58
|
Object.assign(msg, { ctx });
|
|
58
59
|
}
|
|
59
|
-
|
|
60
|
-
log[msg.level || __1.DebugLogLevel.info](...[msg.items, msg.kv, msg.attachments, msg.mentions].filter(Boolean));
|
|
61
|
-
}
|
|
60
|
+
this.cfg.logger.log(...[msg.items, msg.kv, msg.attachments, msg.mentions].filter(Boolean));
|
|
62
61
|
if (!webhookUrl)
|
|
63
62
|
return;
|
|
64
63
|
// Transform msg.kv into msg.attachments
|
|
65
64
|
if (msg.kv) {
|
|
66
|
-
|
|
65
|
+
;
|
|
66
|
+
(msg.attachments || (msg.attachments = [])).push({ fields: this.kvToFields(msg.kv) });
|
|
67
67
|
delete msg.kv; // to not pass it all the way to Slack Api
|
|
68
68
|
}
|
|
69
69
|
let text;
|
|
70
|
-
const inspectOpt = {
|
|
71
|
-
...INSPECT_OPT,
|
|
72
|
-
...msg.inspectOptions,
|
|
73
|
-
};
|
|
74
70
|
// Array has a special treatment here
|
|
75
71
|
if (Array.isArray(msg.items)) {
|
|
76
|
-
text = msg.items.map(t => (0, __1.inspectAny)(t,
|
|
72
|
+
text = msg.items.map(t => (0, __1.inspectAny)(t, inspectOptions)).join('\n');
|
|
77
73
|
}
|
|
78
74
|
else {
|
|
79
|
-
text = (0, __1.inspectAny)(msg.items,
|
|
75
|
+
text = (0, __1.inspectAny)(msg.items, inspectOptions);
|
|
80
76
|
}
|
|
81
77
|
// Wrap in markdown-text-block if it's anything but plain String
|
|
82
78
|
if (typeof msg.items !== 'string') {
|
|
@@ -88,22 +84,18 @@ class SlackService {
|
|
|
88
84
|
const prefix = await messagePrefixHook(msg);
|
|
89
85
|
if (prefix === null)
|
|
90
86
|
return; // filtered out!
|
|
91
|
-
const json = {
|
|
92
|
-
...DEFAULTS
|
|
87
|
+
const json = (0, js_lib_1._omit)({
|
|
88
|
+
...DEFAULTS,
|
|
93
89
|
...this.cfg.defaults,
|
|
94
90
|
...msg,
|
|
95
91
|
// Text with Prefix
|
|
96
92
|
text: [prefix.join(': '), text].filter(Boolean).join('\n'),
|
|
97
|
-
};
|
|
98
|
-
// they're not needed in the json payload
|
|
99
|
-
delete json['items'];
|
|
100
|
-
delete json['ctx'];
|
|
101
|
-
delete json['noLog'];
|
|
102
|
-
json.channel = (this.cfg.channelByLevel || {})[msg.level] || json.channel;
|
|
93
|
+
}, ['items', 'ctx']);
|
|
103
94
|
await got_1.default
|
|
104
95
|
.post(webhookUrl, {
|
|
105
96
|
json,
|
|
106
97
|
responseType: 'text',
|
|
98
|
+
timeout: 90000,
|
|
107
99
|
})
|
|
108
100
|
.catch(err => {
|
|
109
101
|
// ignore (unless throwOnError is set)
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { StringMap } from '@naturalcycles/js-lib';
|
|
2
|
-
import {
|
|
1
|
+
import { CommonLogger, StringMap } from '@naturalcycles/js-lib';
|
|
2
|
+
import { InspectAnyOptions } from '..';
|
|
3
3
|
/**
|
|
4
4
|
* Properties that exists both in SlackApiBody (as per Slack API) and SlackMessage (our abstraction).
|
|
5
5
|
*/
|
|
@@ -36,7 +36,6 @@ export interface SlackMessage<CTX = any> extends SlackMessageProps {
|
|
|
36
36
|
* Optional "context object", to be used by `messagePrefixHook`.
|
|
37
37
|
*/
|
|
38
38
|
ctx?: CTX;
|
|
39
|
-
level?: DebugLogLevel;
|
|
40
39
|
/**
|
|
41
40
|
* Keys-values will be rendered as MessageAttachment with Fields
|
|
42
41
|
*/
|
|
@@ -46,21 +45,11 @@ export interface SlackMessage<CTX = any> extends SlackMessageProps {
|
|
|
46
45
|
*/
|
|
47
46
|
mentions?: string[];
|
|
48
47
|
/**
|
|
49
|
-
* @default false
|
|
50
48
|
* By default it ignores possible errors from slack
|
|
51
|
-
|
|
52
|
-
throwOnError?: boolean;
|
|
53
|
-
/**
|
|
49
|
+
*
|
|
54
50
|
* @default false
|
|
55
|
-
* Skips logging message
|
|
56
|
-
*/
|
|
57
|
-
noLog?: boolean;
|
|
58
|
-
/**
|
|
59
|
-
* Defaults to:
|
|
60
|
-
* includeErrorData: true
|
|
61
|
-
* includeErrorStack: true
|
|
62
51
|
*/
|
|
63
|
-
|
|
52
|
+
throwOnError?: boolean;
|
|
64
53
|
}
|
|
65
54
|
export interface SlackAttachmentField {
|
|
66
55
|
title: string;
|
|
@@ -96,15 +85,21 @@ export interface SlackServiceCfg<CTX = any> {
|
|
|
96
85
|
*/
|
|
97
86
|
webhookUrl?: string;
|
|
98
87
|
defaults?: Partial<SlackMessage>;
|
|
99
|
-
/**
|
|
100
|
-
* Override channel when msg.level is set.
|
|
101
|
-
* key: DebugLogLevel
|
|
102
|
-
* value: channel name to send message to
|
|
103
|
-
*/
|
|
104
|
-
channelByLevel?: StringMap;
|
|
105
88
|
/**
|
|
106
89
|
* Function to return an array of "prefix tokens" (will be joined by ': ').
|
|
107
90
|
* Allows to skip (filter out) the message by returning `null`.
|
|
108
91
|
*/
|
|
109
92
|
messagePrefixHook: SlackMessagePrefixHook<CTX>;
|
|
93
|
+
/**
|
|
94
|
+
* By default SlackService logs every message to console.log
|
|
95
|
+
* Pass another logger if needed.
|
|
96
|
+
* Pass `noopLogger` to suppress logging completely.
|
|
97
|
+
*/
|
|
98
|
+
logger: CommonLogger;
|
|
99
|
+
/**
|
|
100
|
+
* Defaults to:
|
|
101
|
+
* includeErrorData: true
|
|
102
|
+
* includeErrorStack: true
|
|
103
|
+
*/
|
|
104
|
+
inspectOptions: InspectAnyOptions;
|
|
110
105
|
}
|
|
@@ -16,6 +16,10 @@ export interface TransformToNDJsonOptions {
|
|
|
16
16
|
* @default `\n`
|
|
17
17
|
*/
|
|
18
18
|
separator?: string;
|
|
19
|
+
/**
|
|
20
|
+
* @experimental
|
|
21
|
+
*/
|
|
22
|
+
useFlatstr?: boolean;
|
|
19
23
|
}
|
|
20
24
|
/**
|
|
21
25
|
* Transforms objects (objectMode=true) into chunks \n-terminated JSON strings (readableObjectMode=false).
|
|
@@ -7,7 +7,7 @@ const js_lib_1 = require("@naturalcycles/js-lib");
|
|
|
7
7
|
* Transforms objects (objectMode=true) into chunks \n-terminated JSON strings (readableObjectMode=false).
|
|
8
8
|
*/
|
|
9
9
|
function transformToNDJson(opt = {}) {
|
|
10
|
-
const { strict = true, separator = '\n', sortObjects = false } = opt;
|
|
10
|
+
const { strict = true, separator = '\n', sortObjects = false, useFlatstr = false } = opt;
|
|
11
11
|
return new stream_1.Transform({
|
|
12
12
|
objectMode: true,
|
|
13
13
|
readableObjectMode: false,
|
|
@@ -16,7 +16,12 @@ function transformToNDJson(opt = {}) {
|
|
|
16
16
|
if (sortObjects) {
|
|
17
17
|
chunk = (0, js_lib_1._sortObjectDeep)(chunk);
|
|
18
18
|
}
|
|
19
|
-
|
|
19
|
+
if (useFlatstr) {
|
|
20
|
+
cb(null, flatstr(JSON.stringify(chunk) + separator));
|
|
21
|
+
}
|
|
22
|
+
else {
|
|
23
|
+
cb(null, JSON.stringify(chunk) + separator);
|
|
24
|
+
}
|
|
20
25
|
}
|
|
21
26
|
catch (err) {
|
|
22
27
|
console.error(err);
|
|
@@ -31,3 +36,11 @@ function transformToNDJson(opt = {}) {
|
|
|
31
36
|
});
|
|
32
37
|
}
|
|
33
38
|
exports.transformToNDJson = transformToNDJson;
|
|
39
|
+
/**
|
|
40
|
+
* Based on: https://github.com/davidmarkclements/flatstr/blob/master/index.js
|
|
41
|
+
*/
|
|
42
|
+
function flatstr(s) {
|
|
43
|
+
// eslint-disable-next-line
|
|
44
|
+
s | 0;
|
|
45
|
+
return s;
|
|
46
|
+
}
|
|
@@ -47,7 +47,7 @@ function transformLogProgress(opt = {}) {
|
|
|
47
47
|
const mem = process.memoryUsage();
|
|
48
48
|
const now = Date.now();
|
|
49
49
|
const batchedProgress = progress * batchSize;
|
|
50
|
-
const lastRPS = (processedLastSecond *
|
|
50
|
+
const lastRPS = (processedLastSecond * batchSize) / ((now - lastSecondStarted) / 1000) || 0;
|
|
51
51
|
const rpsTotal = Math.round(batchedProgress / ((now - started) / 1000)) || 0;
|
|
52
52
|
lastSecondStarted = now;
|
|
53
53
|
processedLastSecond = 0;
|
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
2
|
import { InspectOptions } from 'util';
|
|
3
|
-
import { StringifyAnyOptions } from '@naturalcycles/js-lib';
|
|
3
|
+
import { StringifyAnyOptions, JsonStringifyFunction } from '@naturalcycles/js-lib';
|
|
4
4
|
export interface InspectAnyOptions extends StringifyAnyOptions, InspectOptions {
|
|
5
5
|
}
|
|
6
|
+
/**
|
|
7
|
+
* Just a convenience export of a const that fulfills the JsonStringifyFunction interface.
|
|
8
|
+
*/
|
|
9
|
+
export declare const inspectAnyStringifyFn: JsonStringifyFunction;
|
|
6
10
|
/**
|
|
7
11
|
* Transforms ANY to human-readable string (via util.inspect mainly).
|
|
8
12
|
* Safe (no error throwing).
|
|
@@ -1,12 +1,17 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.inspectAny = void 0;
|
|
3
|
+
exports.inspectAny = exports.inspectAnyStringifyFn = void 0;
|
|
4
4
|
const util_1 = require("util");
|
|
5
5
|
const js_lib_1 = require("@naturalcycles/js-lib");
|
|
6
6
|
const INSPECT_OPT = {
|
|
7
7
|
breakLength: 80,
|
|
8
8
|
depth: 10, // default: 2
|
|
9
9
|
};
|
|
10
|
+
/**
|
|
11
|
+
* Just a convenience export of a const that fulfills the JsonStringifyFunction interface.
|
|
12
|
+
*/
|
|
13
|
+
const inspectAnyStringifyFn = obj => inspectAny(obj);
|
|
14
|
+
exports.inspectAnyStringifyFn = inspectAnyStringifyFn;
|
|
10
15
|
/**
|
|
11
16
|
* Transforms ANY to human-readable string (via util.inspect mainly).
|
|
12
17
|
* Safe (no error throwing).
|
|
@@ -5,20 +5,6 @@ const js_lib_1 = require("@naturalcycles/js-lib");
|
|
|
5
5
|
class AjvValidationError extends js_lib_1.AppError {
|
|
6
6
|
constructor(message, data) {
|
|
7
7
|
super(message, data);
|
|
8
|
-
this.constructor = AjvValidationError;
|
|
9
|
-
this.__proto__ = AjvValidationError.prototype;
|
|
10
|
-
Object.defineProperty(this, 'name', {
|
|
11
|
-
value: this.constructor.name,
|
|
12
|
-
configurable: true,
|
|
13
|
-
});
|
|
14
|
-
if (Error.captureStackTrace) {
|
|
15
|
-
Error.captureStackTrace(this, this.constructor);
|
|
16
|
-
}
|
|
17
|
-
else {
|
|
18
|
-
Object.defineProperty(this, 'stack', {
|
|
19
|
-
value: new Error().stack, // eslint-disable-line unicorn/error-message
|
|
20
|
-
});
|
|
21
|
-
}
|
|
22
8
|
}
|
|
23
9
|
}
|
|
24
10
|
exports.AjvValidationError = AjvValidationError;
|
|
@@ -5,20 +5,6 @@ const js_lib_1 = require("@naturalcycles/js-lib");
|
|
|
5
5
|
class JoiValidationError extends js_lib_1.AppError {
|
|
6
6
|
constructor(message, data) {
|
|
7
7
|
super(message, data);
|
|
8
|
-
this.constructor = JoiValidationError;
|
|
9
|
-
this.__proto__ = JoiValidationError.prototype;
|
|
10
|
-
Object.defineProperty(this, 'name', {
|
|
11
|
-
value: this.constructor.name,
|
|
12
|
-
configurable: true,
|
|
13
|
-
});
|
|
14
|
-
if (Error.captureStackTrace) {
|
|
15
|
-
Error.captureStackTrace(this, this.constructor);
|
|
16
|
-
}
|
|
17
|
-
else {
|
|
18
|
-
Object.defineProperty(this, 'stack', {
|
|
19
|
-
value: new Error().stack, // eslint-disable-line unicorn/error-message
|
|
20
|
-
});
|
|
21
|
-
}
|
|
22
8
|
}
|
|
23
9
|
}
|
|
24
10
|
exports.JoiValidationError = JoiValidationError;
|
|
@@ -130,6 +130,7 @@ function createError(value, err, objectName) {
|
|
|
130
130
|
// Make annotation non-enumerable, to not get it automatically printed,
|
|
131
131
|
// but still accessible
|
|
132
132
|
Object.defineProperty(data, 'annotation', {
|
|
133
|
+
writable: true,
|
|
133
134
|
configurable: true,
|
|
134
135
|
enumerable: false,
|
|
135
136
|
value: annotation,
|
package/package.json
CHANGED
package/src/got/getGot.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { URL } from 'url'
|
|
2
|
-
import {
|
|
2
|
+
import { _since } from '@naturalcycles/js-lib'
|
|
3
3
|
import got, { AfterResponseHook, BeforeErrorHook, BeforeRequestHook, Got, HTTPError } from 'got'
|
|
4
4
|
import { inspectAny } from '..'
|
|
5
5
|
import { dimGrey, grey, red, yellow } from '../colors'
|
|
@@ -13,6 +13,8 @@ import { GetGotOptions, GotRequestContext } from './got.model'
|
|
|
13
13
|
* 3. Reasonable defaults(tm), e.g non-infinite Timeout
|
|
14
14
|
*/
|
|
15
15
|
export function getGot(opt: GetGotOptions = {}): Got {
|
|
16
|
+
opt.logger ||= console
|
|
17
|
+
|
|
16
18
|
return got.extend({
|
|
17
19
|
// Most-important is to set to anything non-empty (so, requests don't "hang" by default).
|
|
18
20
|
// Should be long enough to handle for slow responses from scaled cloud APIs in times of spikes
|
|
@@ -99,7 +101,7 @@ function gotBeforeRequestHook(opt: GetGotOptions): BeforeRequestHook {
|
|
|
99
101
|
|
|
100
102
|
if (opt.logStart) {
|
|
101
103
|
const shortUrl = getShortUrl(opt, options.url, options.prefixUrl)
|
|
102
|
-
|
|
104
|
+
opt.logger!.log([dimGrey(' >>'), dimGrey(options.method), grey(shortUrl)].join(' '))
|
|
103
105
|
}
|
|
104
106
|
}
|
|
105
107
|
}
|
|
@@ -113,7 +115,7 @@ function gotAfterResponseHook(opt: GetGotOptions = {}): AfterResponseHook {
|
|
|
113
115
|
const { url, prefixUrl, method } = resp.request.options
|
|
114
116
|
const shortUrl = getShortUrl(opt, url, prefixUrl)
|
|
115
117
|
|
|
116
|
-
|
|
118
|
+
opt.logger!.log(
|
|
117
119
|
[
|
|
118
120
|
dimGrey(' <<'),
|
|
119
121
|
coloredHttpCode(resp.statusCode),
|
|
@@ -129,7 +131,7 @@ function gotAfterResponseHook(opt: GetGotOptions = {}): AfterResponseHook {
|
|
|
129
131
|
|
|
130
132
|
// Error responses are not logged, cause they're included in Error message already
|
|
131
133
|
if (opt.logResponse && success) {
|
|
132
|
-
|
|
134
|
+
opt.logger!.log(inspectAny(resp.body, { maxLen: opt.maxResponseLength }))
|
|
133
135
|
}
|
|
134
136
|
|
|
135
137
|
return resp
|
package/src/got/got.model.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { AnyObject } from '@naturalcycles/js-lib'
|
|
1
|
+
import { AnyObject, CommonLogger } from '@naturalcycles/js-lib'
|
|
2
2
|
import type { Options } from 'got'
|
|
3
3
|
|
|
4
4
|
export interface GetGotOptions extends Options {
|
|
@@ -33,6 +33,11 @@ export interface GetGotOptions extends Options {
|
|
|
33
33
|
*/
|
|
34
34
|
logWithSearchParams?: boolean
|
|
35
35
|
|
|
36
|
+
/**
|
|
37
|
+
* Defaults to `console`
|
|
38
|
+
*/
|
|
39
|
+
logger?: CommonLogger
|
|
40
|
+
|
|
36
41
|
/**
|
|
37
42
|
* Max length of response object before it's truncated.
|
|
38
43
|
*
|
package/src/index.ts
CHANGED
|
@@ -7,7 +7,7 @@ import { tableDiff, TableDiffOptions } from './diff/tableDiff'
|
|
|
7
7
|
import { getGot } from './got/getGot'
|
|
8
8
|
import { GetGotOptions } from './got/got.model'
|
|
9
9
|
import { memoryUsage, memoryUsageFull, processSharedUtil } from './infra/process.util'
|
|
10
|
-
import { Debug,
|
|
10
|
+
import { Debug, IDebug, IDebugger } from './log/debug'
|
|
11
11
|
import {
|
|
12
12
|
base64ToBuffer,
|
|
13
13
|
base64ToString,
|
|
@@ -108,7 +108,7 @@ import { writableForEach } from './stream/writable/writableForEach'
|
|
|
108
108
|
import { writableFork } from './stream/writable/writableFork'
|
|
109
109
|
import { writablePushToArray } from './stream/writable/writablePushToArray'
|
|
110
110
|
import { writableVoid } from './stream/writable/writableVoid'
|
|
111
|
-
import { inspectAny, InspectAnyOptions } from './string/inspectAny'
|
|
111
|
+
import { inspectAny, InspectAnyOptions, inspectAnyStringifyFn } from './string/inspectAny'
|
|
112
112
|
import { requireEnvKeys, requireFileToExist } from './util/env.util'
|
|
113
113
|
import { LRUMemoCache } from './util/lruMemoCache'
|
|
114
114
|
import {
|
|
@@ -287,7 +287,6 @@ export {
|
|
|
287
287
|
bufferToBase64,
|
|
288
288
|
base64ToBuffer,
|
|
289
289
|
Debug,
|
|
290
|
-
DebugLogLevel,
|
|
291
290
|
getSecretMap,
|
|
292
291
|
setSecretMap,
|
|
293
292
|
loadSecretsFromEnv,
|
|
@@ -340,6 +339,7 @@ export {
|
|
|
340
339
|
BaseWorkerClass,
|
|
341
340
|
tableDiff,
|
|
342
341
|
inspectAny,
|
|
342
|
+
inspectAnyStringifyFn,
|
|
343
343
|
getGot,
|
|
344
344
|
HTTPError,
|
|
345
345
|
TimeoutError,
|
package/src/log/debug.ts
CHANGED
|
@@ -20,10 +20,6 @@ export interface DebugFormatters {
|
|
|
20
20
|
export interface IDebugger {
|
|
21
21
|
// (formatter: any, ...args: any[]): void;
|
|
22
22
|
(...args: any[]): void
|
|
23
|
-
debug: (...args: any[]) => void
|
|
24
|
-
info: (...args: any[]) => void // alias to just log()
|
|
25
|
-
warn: (...args: any[]) => void
|
|
26
|
-
error: (...args: any[]) => void
|
|
27
23
|
|
|
28
24
|
color: string
|
|
29
25
|
enabled: boolean
|
|
@@ -33,35 +29,15 @@ export interface IDebugger {
|
|
|
33
29
|
// extend: (namespace: string, delimiter?: string) => IDebugger
|
|
34
30
|
}
|
|
35
31
|
|
|
36
|
-
export enum DebugLogLevel {
|
|
37
|
-
debug = 'debug',
|
|
38
|
-
info = 'info',
|
|
39
|
-
warn = 'warn',
|
|
40
|
-
error = 'error',
|
|
41
|
-
}
|
|
42
|
-
|
|
43
32
|
const originalDebug = require('debug') as IDebug
|
|
44
33
|
|
|
45
34
|
// eslint-disable-next-line @typescript-eslint/naming-convention
|
|
46
35
|
export const Debug = ((namespace: string) => {
|
|
47
36
|
const instance = originalDebug(namespace)
|
|
48
37
|
instance.log = console.log.bind(console) // this enables colors for objects
|
|
49
|
-
instance.info = instance.bind(instance)
|
|
50
|
-
|
|
51
|
-
const instanceDebug = originalDebug([namespace, 'debug'].join(':'))
|
|
52
|
-
instanceDebug.log = console.debug.bind(console)
|
|
53
|
-
instance.debug = instanceDebug.bind(instanceDebug)
|
|
54
|
-
|
|
55
|
-
const instanceWarn = originalDebug([namespace, 'warn'].join(':'))
|
|
56
|
-
instanceWarn.log = console.warn.bind(console)
|
|
57
|
-
instance.warn = instanceWarn.bind(instanceWarn)
|
|
58
|
-
|
|
59
|
-
const instanceError = originalDebug([namespace, 'error'].join(':'))
|
|
60
|
-
instanceError.log = console.error.bind(console)
|
|
61
|
-
instance.error = instanceError.bind(instanceError)
|
|
62
|
-
|
|
63
38
|
return instance
|
|
64
39
|
}) as IDebug
|
|
40
|
+
|
|
65
41
|
Debug.coerce = originalDebug.coerce.bind(originalDebug)
|
|
66
42
|
Debug.disable = originalDebug.disable.bind(originalDebug)
|
|
67
43
|
Debug.enable = originalDebug.enable.bind(originalDebug)
|
|
@@ -1,13 +1,10 @@
|
|
|
1
1
|
import * as fs from 'fs'
|
|
2
2
|
import { StringMap } from '@naturalcycles/js-lib'
|
|
3
|
-
import { base64ToString
|
|
3
|
+
import { base64ToString } from '..'
|
|
4
4
|
import { decryptRandomIVBuffer } from './crypto.util'
|
|
5
5
|
|
|
6
6
|
let loaded = false
|
|
7
7
|
|
|
8
|
-
// it's wrapped to be able to pipe console.* to Stackdriver
|
|
9
|
-
const getLog = () => Debug('nc:nodejs-lib:secret')
|
|
10
|
-
|
|
11
8
|
const secretMap: StringMap = {}
|
|
12
9
|
|
|
13
10
|
/**
|
|
@@ -29,7 +26,7 @@ export function loadSecretsFromEnv(): void {
|
|
|
29
26
|
})
|
|
30
27
|
|
|
31
28
|
loaded = true
|
|
32
|
-
|
|
29
|
+
console.log(
|
|
33
30
|
`${Object.keys(secrets).length} secret(s) loaded from process.env: ${Object.keys(secrets).join(
|
|
34
31
|
', ',
|
|
35
32
|
)}`,
|
|
@@ -69,7 +66,7 @@ export function loadSecretsFromJsonFile(filePath: string, SECRET_ENCRYPTION_KEY?
|
|
|
69
66
|
Object.entries(secrets).forEach(([k, v]) => (secretMap[k.toUpperCase()] = v))
|
|
70
67
|
|
|
71
68
|
loaded = true
|
|
72
|
-
|
|
69
|
+
console.log(
|
|
73
70
|
`${Object.keys(secrets).length} secret(s) loaded from ${filePath}: ${Object.keys(secrets)
|
|
74
71
|
.map(s => s.toUpperCase())
|
|
75
72
|
.join(', ')}`,
|
|
@@ -105,7 +102,7 @@ export function getSecretMap(): StringMap {
|
|
|
105
102
|
export function setSecretMap(map: StringMap): void {
|
|
106
103
|
Object.keys(secretMap).forEach(k => delete secretMap[k])
|
|
107
104
|
Object.entries(map).forEach(([k, v]) => (secretMap[k.toUpperCase()] = v))
|
|
108
|
-
|
|
105
|
+
console.log(
|
|
109
106
|
`setSecretMap set ${Object.keys(secretMap).length} secret(s): ${Object.keys(map)
|
|
110
107
|
.map(s => s.toUpperCase())
|
|
111
108
|
.join(', ')}`,
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { StringMap } from '@naturalcycles/js-lib'
|
|
2
|
-
import {
|
|
1
|
+
import { CommonLogger, StringMap } from '@naturalcycles/js-lib'
|
|
2
|
+
import { InspectAnyOptions } from '..'
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
5
|
* Properties that exists both in SlackApiBody (as per Slack API) and SlackMessage (our abstraction).
|
|
@@ -44,8 +44,6 @@ export interface SlackMessage<CTX = any> extends SlackMessageProps {
|
|
|
44
44
|
*/
|
|
45
45
|
ctx?: CTX
|
|
46
46
|
|
|
47
|
-
level?: DebugLogLevel
|
|
48
|
-
|
|
49
47
|
/**
|
|
50
48
|
* Keys-values will be rendered as MessageAttachment with Fields
|
|
51
49
|
*/
|
|
@@ -57,23 +55,11 @@ export interface SlackMessage<CTX = any> extends SlackMessageProps {
|
|
|
57
55
|
mentions?: string[]
|
|
58
56
|
|
|
59
57
|
/**
|
|
60
|
-
* @default false
|
|
61
58
|
* By default it ignores possible errors from slack
|
|
62
|
-
|
|
63
|
-
throwOnError?: boolean
|
|
64
|
-
|
|
65
|
-
/**
|
|
59
|
+
*
|
|
66
60
|
* @default false
|
|
67
|
-
* Skips logging message
|
|
68
|
-
*/
|
|
69
|
-
noLog?: boolean
|
|
70
|
-
|
|
71
|
-
/**
|
|
72
|
-
* Defaults to:
|
|
73
|
-
* includeErrorData: true
|
|
74
|
-
* includeErrorStack: true
|
|
75
61
|
*/
|
|
76
|
-
|
|
62
|
+
throwOnError?: boolean
|
|
77
63
|
}
|
|
78
64
|
|
|
79
65
|
export interface SlackAttachmentField {
|
|
@@ -120,16 +106,23 @@ export interface SlackServiceCfg<CTX = any> {
|
|
|
120
106
|
|
|
121
107
|
defaults?: Partial<SlackMessage>
|
|
122
108
|
|
|
123
|
-
/**
|
|
124
|
-
* Override channel when msg.level is set.
|
|
125
|
-
* key: DebugLogLevel
|
|
126
|
-
* value: channel name to send message to
|
|
127
|
-
*/
|
|
128
|
-
channelByLevel?: StringMap
|
|
129
|
-
|
|
130
109
|
/**
|
|
131
110
|
* Function to return an array of "prefix tokens" (will be joined by ': ').
|
|
132
111
|
* Allows to skip (filter out) the message by returning `null`.
|
|
133
112
|
*/
|
|
134
113
|
messagePrefixHook: SlackMessagePrefixHook<CTX>
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* By default SlackService logs every message to console.log
|
|
117
|
+
* Pass another logger if needed.
|
|
118
|
+
* Pass `noopLogger` to suppress logging completely.
|
|
119
|
+
*/
|
|
120
|
+
logger: CommonLogger
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* Defaults to:
|
|
124
|
+
* includeErrorData: true
|
|
125
|
+
* includeErrorStack: true
|
|
126
|
+
*/
|
|
127
|
+
inspectOptions: InspectAnyOptions
|
|
135
128
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { StringMap } from '@naturalcycles/js-lib'
|
|
1
|
+
import { _omit, StringMap } from '@naturalcycles/js-lib'
|
|
2
2
|
import { dayjs } from '@naturalcycles/time-lib'
|
|
3
3
|
import got from 'got'
|
|
4
|
-
import {
|
|
4
|
+
import { inspectAny, InspectAnyOptions } from '..'
|
|
5
5
|
import {
|
|
6
6
|
SlackApiBody,
|
|
7
7
|
SlackAttachmentField,
|
|
@@ -11,12 +11,12 @@ import {
|
|
|
11
11
|
|
|
12
12
|
const GAE = !!process.env['GAE_INSTANCE']
|
|
13
13
|
|
|
14
|
-
const DEFAULTS
|
|
14
|
+
const DEFAULTS: SlackMessage = {
|
|
15
15
|
username: 'bot',
|
|
16
16
|
channel: '#log',
|
|
17
17
|
icon_emoji: ':spider_web:',
|
|
18
18
|
items: 'no text',
|
|
19
|
-
}
|
|
19
|
+
}
|
|
20
20
|
|
|
21
21
|
const INSPECT_OPT: InspectAnyOptions = {
|
|
22
22
|
colors: false,
|
|
@@ -24,8 +24,6 @@ const INSPECT_OPT: InspectAnyOptions = {
|
|
|
24
24
|
includeErrorStack: true,
|
|
25
25
|
}
|
|
26
26
|
|
|
27
|
-
const log = Debug('nc:nodejs-lib:slack')
|
|
28
|
-
|
|
29
27
|
/**
|
|
30
28
|
* Has 2 main methods:
|
|
31
29
|
*
|
|
@@ -42,7 +40,12 @@ export class SlackService<CTX = any> {
|
|
|
42
40
|
constructor(cfg: Partial<SlackServiceCfg<CTX>>) {
|
|
43
41
|
this.cfg = {
|
|
44
42
|
messagePrefixHook: slackDefaultMessagePrefixHook,
|
|
43
|
+
logger: console,
|
|
45
44
|
...cfg,
|
|
45
|
+
inspectOptions: {
|
|
46
|
+
...INSPECT_OPT,
|
|
47
|
+
...cfg.inspectOptions,
|
|
48
|
+
},
|
|
46
49
|
}
|
|
47
50
|
}
|
|
48
51
|
|
|
@@ -58,47 +61,34 @@ export class SlackService<CTX = any> {
|
|
|
58
61
|
})
|
|
59
62
|
}
|
|
60
63
|
|
|
61
|
-
async send(
|
|
62
|
-
const { webhookUrl, messagePrefixHook } = this.cfg
|
|
64
|
+
async send(input: SlackMessage<CTX> | string, ctx?: CTX): Promise<void> {
|
|
65
|
+
const { webhookUrl, messagePrefixHook, inspectOptions } = this.cfg
|
|
63
66
|
|
|
64
67
|
// If String is passed as first argument - just transform it to a full SlackMessage
|
|
65
|
-
|
|
66
|
-
msg = {
|
|
67
|
-
items: msg,
|
|
68
|
-
}
|
|
69
|
-
}
|
|
68
|
+
const msg = typeof input === 'string' ? { items: input } : input
|
|
70
69
|
|
|
71
70
|
if (ctx !== undefined) {
|
|
72
71
|
Object.assign(msg, { ctx })
|
|
73
72
|
}
|
|
74
73
|
|
|
75
|
-
|
|
76
|
-
log[msg.level || DebugLogLevel.info](
|
|
77
|
-
...[msg.items, msg.kv, msg.attachments, msg.mentions].filter(Boolean),
|
|
78
|
-
)
|
|
79
|
-
}
|
|
74
|
+
this.cfg.logger.log(...[msg.items, msg.kv, msg.attachments, msg.mentions].filter(Boolean))
|
|
80
75
|
|
|
81
76
|
if (!webhookUrl) return
|
|
82
77
|
|
|
83
78
|
// Transform msg.kv into msg.attachments
|
|
84
79
|
if (msg.kv) {
|
|
85
|
-
|
|
80
|
+
;(msg.attachments ||= []).push({ fields: this.kvToFields(msg.kv) })
|
|
86
81
|
|
|
87
82
|
delete msg.kv // to not pass it all the way to Slack Api
|
|
88
83
|
}
|
|
89
84
|
|
|
90
85
|
let text: string
|
|
91
86
|
|
|
92
|
-
const inspectOpt = {
|
|
93
|
-
...INSPECT_OPT,
|
|
94
|
-
...msg.inspectOptions,
|
|
95
|
-
}
|
|
96
|
-
|
|
97
87
|
// Array has a special treatment here
|
|
98
88
|
if (Array.isArray(msg.items)) {
|
|
99
|
-
text = msg.items.map(t => inspectAny(t,
|
|
89
|
+
text = msg.items.map(t => inspectAny(t, inspectOptions)).join('\n')
|
|
100
90
|
} else {
|
|
101
|
-
text = inspectAny(msg.items,
|
|
91
|
+
text = inspectAny(msg.items, inspectOptions)
|
|
102
92
|
}
|
|
103
93
|
|
|
104
94
|
// Wrap in markdown-text-block if it's anything but plain String
|
|
@@ -113,29 +103,26 @@ export class SlackService<CTX = any> {
|
|
|
113
103
|
const prefix = await messagePrefixHook(msg)
|
|
114
104
|
if (prefix === null) return // filtered out!
|
|
115
105
|
|
|
116
|
-
const json: SlackApiBody =
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
delete json['ctx']
|
|
127
|
-
delete json['noLog']
|
|
128
|
-
|
|
129
|
-
json.channel = (this.cfg.channelByLevel || {})[msg.level!] || json.channel
|
|
106
|
+
const json: SlackApiBody = _omit(
|
|
107
|
+
{
|
|
108
|
+
...DEFAULTS,
|
|
109
|
+
...this.cfg.defaults,
|
|
110
|
+
...msg,
|
|
111
|
+
// Text with Prefix
|
|
112
|
+
text: [prefix.join(': '), text].filter(Boolean).join('\n'),
|
|
113
|
+
},
|
|
114
|
+
['items', 'ctx'],
|
|
115
|
+
)
|
|
130
116
|
|
|
131
117
|
await got
|
|
132
118
|
.post(webhookUrl, {
|
|
133
119
|
json,
|
|
134
120
|
responseType: 'text',
|
|
121
|
+
timeout: 90_000,
|
|
135
122
|
})
|
|
136
123
|
.catch(err => {
|
|
137
124
|
// ignore (unless throwOnError is set)
|
|
138
|
-
if (
|
|
125
|
+
if (msg.throwOnError) throw err
|
|
139
126
|
})
|
|
140
127
|
}
|
|
141
128
|
|
|
@@ -21,6 +21,11 @@ export interface TransformToNDJsonOptions {
|
|
|
21
21
|
* @default `\n`
|
|
22
22
|
*/
|
|
23
23
|
separator?: string
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* @experimental
|
|
27
|
+
*/
|
|
28
|
+
useFlatstr?: boolean
|
|
24
29
|
}
|
|
25
30
|
|
|
26
31
|
/**
|
|
@@ -29,7 +34,7 @@ export interface TransformToNDJsonOptions {
|
|
|
29
34
|
export function transformToNDJson<IN = any>(
|
|
30
35
|
opt: TransformToNDJsonOptions = {},
|
|
31
36
|
): TransformTyped<IN, string> {
|
|
32
|
-
const { strict = true, separator = '\n', sortObjects = false } = opt
|
|
37
|
+
const { strict = true, separator = '\n', sortObjects = false, useFlatstr = false } = opt
|
|
33
38
|
|
|
34
39
|
return new Transform({
|
|
35
40
|
objectMode: true,
|
|
@@ -39,7 +44,12 @@ export function transformToNDJson<IN = any>(
|
|
|
39
44
|
if (sortObjects) {
|
|
40
45
|
chunk = _sortObjectDeep(chunk as any)
|
|
41
46
|
}
|
|
42
|
-
|
|
47
|
+
|
|
48
|
+
if (useFlatstr) {
|
|
49
|
+
cb(null, flatstr(JSON.stringify(chunk) + separator))
|
|
50
|
+
} else {
|
|
51
|
+
cb(null, JSON.stringify(chunk) + separator)
|
|
52
|
+
}
|
|
43
53
|
} catch (err) {
|
|
44
54
|
console.error(err)
|
|
45
55
|
|
|
@@ -52,3 +62,12 @@ export function transformToNDJson<IN = any>(
|
|
|
52
62
|
},
|
|
53
63
|
})
|
|
54
64
|
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Based on: https://github.com/davidmarkclements/flatstr/blob/master/index.js
|
|
68
|
+
*/
|
|
69
|
+
function flatstr(s: any): string {
|
|
70
|
+
// eslint-disable-next-line
|
|
71
|
+
s | 0
|
|
72
|
+
return s
|
|
73
|
+
}
|
|
@@ -164,8 +164,7 @@ export function transformLogProgress<IN = any>(
|
|
|
164
164
|
|
|
165
165
|
const now = Date.now()
|
|
166
166
|
const batchedProgress = progress * batchSize
|
|
167
|
-
const lastRPS =
|
|
168
|
-
(processedLastSecond * batchedProgress) / ((now - lastSecondStarted) / 1000) || 0
|
|
167
|
+
const lastRPS = (processedLastSecond * batchSize) / ((now - lastSecondStarted) / 1000) || 0
|
|
169
168
|
const rpsTotal = Math.round(batchedProgress / ((now - started) / 1000)) || 0
|
|
170
169
|
lastSecondStarted = now
|
|
171
170
|
processedLastSecond = 0
|
package/src/string/inspectAny.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { inspect, InspectOptions } from 'util'
|
|
2
|
-
import { StringifyAnyOptions, _stringifyAny } from '@naturalcycles/js-lib'
|
|
2
|
+
import { StringifyAnyOptions, _stringifyAny, JsonStringifyFunction } from '@naturalcycles/js-lib'
|
|
3
3
|
|
|
4
4
|
export interface InspectAnyOptions extends StringifyAnyOptions, InspectOptions {}
|
|
5
5
|
|
|
@@ -8,6 +8,11 @@ const INSPECT_OPT: InspectOptions = {
|
|
|
8
8
|
depth: 10, // default: 2
|
|
9
9
|
}
|
|
10
10
|
|
|
11
|
+
/**
|
|
12
|
+
* Just a convenience export of a const that fulfills the JsonStringifyFunction interface.
|
|
13
|
+
*/
|
|
14
|
+
export const inspectAnyStringifyFn: JsonStringifyFunction = obj => inspectAny(obj)
|
|
15
|
+
|
|
11
16
|
/**
|
|
12
17
|
* Transforms ANY to human-readable string (via util.inspect mainly).
|
|
13
18
|
* Safe (no error throwing).
|
|
@@ -10,20 +10,5 @@ export interface AjvValidationErrorData extends ErrorData {
|
|
|
10
10
|
export class AjvValidationError extends AppError<AjvValidationErrorData> {
|
|
11
11
|
constructor(message: string, data: AjvValidationErrorData) {
|
|
12
12
|
super(message, data)
|
|
13
|
-
|
|
14
|
-
this.constructor = AjvValidationError
|
|
15
|
-
;(this as any).__proto__ = AjvValidationError.prototype
|
|
16
|
-
Object.defineProperty(this, 'name', {
|
|
17
|
-
value: this.constructor.name,
|
|
18
|
-
configurable: true,
|
|
19
|
-
})
|
|
20
|
-
|
|
21
|
-
if (Error.captureStackTrace) {
|
|
22
|
-
Error.captureStackTrace(this, this.constructor)
|
|
23
|
-
} else {
|
|
24
|
-
Object.defineProperty(this, 'stack', {
|
|
25
|
-
value: new Error().stack, // eslint-disable-line unicorn/error-message
|
|
26
|
-
})
|
|
27
|
-
}
|
|
28
13
|
}
|
|
29
14
|
}
|
|
@@ -28,20 +28,5 @@ export interface JoiValidationErrorData extends ErrorData {
|
|
|
28
28
|
export class JoiValidationError extends AppError<JoiValidationErrorData> {
|
|
29
29
|
constructor(message: string, data: JoiValidationErrorData) {
|
|
30
30
|
super(message, data)
|
|
31
|
-
|
|
32
|
-
this.constructor = JoiValidationError
|
|
33
|
-
;(this as any).__proto__ = JoiValidationError.prototype
|
|
34
|
-
Object.defineProperty(this, 'name', {
|
|
35
|
-
value: this.constructor.name,
|
|
36
|
-
configurable: true,
|
|
37
|
-
})
|
|
38
|
-
|
|
39
|
-
if (Error.captureStackTrace) {
|
|
40
|
-
Error.captureStackTrace(this, this.constructor)
|
|
41
|
-
} else {
|
|
42
|
-
Object.defineProperty(this, 'stack', {
|
|
43
|
-
value: new Error().stack, // eslint-disable-line unicorn/error-message
|
|
44
|
-
})
|
|
45
|
-
}
|
|
46
31
|
}
|
|
47
32
|
}
|
|
@@ -172,6 +172,7 @@ function createError(value: any, err: ValidationError, objectName?: string): Joi
|
|
|
172
172
|
// Make annotation non-enumerable, to not get it automatically printed,
|
|
173
173
|
// but still accessible
|
|
174
174
|
Object.defineProperty(data, 'annotation', {
|
|
175
|
+
writable: true,
|
|
175
176
|
configurable: true,
|
|
176
177
|
enumerable: false,
|
|
177
178
|
value: annotation,
|