@naturalcycles/nodejs-lib 12.50.0 → 12.51.0

package/dist/security/secret.util.js

@@ -5,8 +5,6 @@ const fs = require("fs");
 const __1 = require("..");
 const crypto_util_1 = require("./crypto.util");
 let loaded = false;
-// it's wrapped to be able to pipe console.* to Stackdriver
-const getLog = () => (0, __1.Debug)('nc:nodejs-lib:secret');
 const secretMap = {};
 /**
  * Loads plaintext secrets from process.env, removes them, stores locally.
@@ -25,7 +23,7 @@ function loadSecretsFromEnv() {
         delete process.env[k];
     });
     loaded = true;
-    getLog()(`${Object.keys(secrets).length} secret(s) loaded from process.env: ${Object.keys(secrets).join(', ')}`);
+    console.log(`${Object.keys(secrets).length} secret(s) loaded from process.env: ${Object.keys(secrets).join(', ')}`);
 }
 exports.loadSecretsFromEnv = loadSecretsFromEnv;
 /**
@@ -58,7 +56,7 @@ function loadSecretsFromJsonFile(filePath, SECRET_ENCRYPTION_KEY) {
     }
     Object.entries(secrets).forEach(([k, v]) => (secretMap[k.toUpperCase()] = v));
     loaded = true;
-    getLog()(`${Object.keys(secrets).length} secret(s) loaded from ${filePath}: ${Object.keys(secrets)
+    console.log(`${Object.keys(secrets).length} secret(s) loaded from ${filePath}: ${Object.keys(secrets)
         .map(s => s.toUpperCase())
         .join(', ')}`);
 }
@@ -91,7 +89,7 @@ exports.getSecretMap = getSecretMap;
 function setSecretMap(map) {
     Object.keys(secretMap).forEach(k => delete secretMap[k]);
     Object.entries(map).forEach(([k, v]) => (secretMap[k.toUpperCase()] = v));
-    getLog()(`setSecretMap set ${Object.keys(secretMap).length} secret(s): ${Object.keys(map)
+    console.log(`setSecretMap set ${Object.keys(secretMap).length} secret(s): ${Object.keys(map)
         .map(s => s.toUpperCase())
         .join(', ')}`);
 }

package/dist/slack/slack.service.d.ts

@@ -19,7 +19,7 @@ export declare class SlackService<CTX = any> {
      * Allows to "log" many things at once, similar to `console.log(one, two, three).
      */
     log(...items: any[]): Promise<void>;
-    send(msg: SlackMessage<CTX> | string, ctx?: CTX): Promise<void>;
+    send(input: SlackMessage<CTX> | string, ctx?: CTX): Promise<void>;
     kvToFields(kv: StringMap<any>): SlackAttachmentField[];
 }
 export declare function slackDefaultMessagePrefixHook(msg: SlackMessage): string[];

package/dist/slack/slack.service.js

@@ -1,22 +1,22 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.slackDefaultMessagePrefixHook = exports.SlackService = void 0;
+const js_lib_1 = require("@naturalcycles/js-lib");
 const time_lib_1 = require("@naturalcycles/time-lib");
 const got_1 = require("got");
 const __1 = require("..");
 const GAE = !!process.env['GAE_INSTANCE'];
-const DEFAULTS = () => ({
+const DEFAULTS = {
     username: 'bot',
     channel: '#log',
     icon_emoji: ':spider_web:',
     items: 'no text',
-});
+};
 const INSPECT_OPT = {
     colors: false,
     includeErrorData: true,
     includeErrorStack: true,
 };
-const log = (0, __1.Debug)('nc:nodejs-lib:slack');
 /**
  * Has 2 main methods:
  *
@@ -33,7 +33,12 @@ class SlackService {
     constructor(cfg) {
         this.cfg = {
             messagePrefixHook: slackDefaultMessagePrefixHook,
+            logger: console,
             ...cfg,
+            inspectOptions: {
+                ...INSPECT_OPT,
+                ...cfg.inspectOptions,
+            },
         };
     }
     /**
@@ -45,38 +50,29 @@ class SlackService {
             items: items.length === 1 ? items[0] : items,
         });
     }
-    async send(msg, ctx) {
-        const { webhookUrl, messagePrefixHook } = this.cfg;
+    async send(input, ctx) {
+        const { webhookUrl, messagePrefixHook, inspectOptions } = this.cfg;
         // If String is passed as first argument - just transform it to a full SlackMessage
-        if (typeof msg === 'string') {
-            msg = {
-                items: msg,
-            };
-        }
+        const msg = typeof input === 'string' ? { items: input } : input;
         if (ctx !== undefined) {
             Object.assign(msg, { ctx });
         }
-        if (!msg.noLog) {
-            log[msg.level || __1.DebugLogLevel.info](...[msg.items, msg.kv, msg.attachments, msg.mentions].filter(Boolean));
-        }
+        this.cfg.logger.log(...[msg.items, msg.kv, msg.attachments, msg.mentions].filter(Boolean));
         if (!webhookUrl)
             return;
         // Transform msg.kv into msg.attachments
         if (msg.kv) {
-            msg.attachments = [...(msg.attachments || []), { fields: this.kvToFields(msg.kv) }];
+            ;
+            (msg.attachments || (msg.attachments = [])).push({ fields: this.kvToFields(msg.kv) });
             delete msg.kv; // to not pass it all the way to Slack Api
         }
         let text;
-        const inspectOpt = {
-            ...INSPECT_OPT,
-            ...msg.inspectOptions,
-        };
         // Array has a special treatment here
         if (Array.isArray(msg.items)) {
-            text = msg.items.map(t => (0, __1.inspectAny)(t, inspectOpt)).join('\n');
+            text = msg.items.map(t => (0, __1.inspectAny)(t, inspectOptions)).join('\n');
         }
         else {
-            text = (0, __1.inspectAny)(msg.items, inspectOpt);
+            text = (0, __1.inspectAny)(msg.items, inspectOptions);
         }
         // Wrap in markdown-text-block if it's anything but plain String
         if (typeof msg.items !== 'string') {
@@ -88,22 +84,18 @@ class SlackService {
         const prefix = await messagePrefixHook(msg);
         if (prefix === null)
             return; // filtered out!
-        const json = {
-            ...DEFAULTS(),
+        const json = (0, js_lib_1._omit)({
+            ...DEFAULTS,
             ...this.cfg.defaults,
             ...msg,
             // Text with Prefix
             text: [prefix.join(': '), text].filter(Boolean).join('\n'),
-        };
-        // they're not needed in the json payload
-        delete json['items'];
-        delete json['ctx'];
-        delete json['noLog'];
-        json.channel = (this.cfg.channelByLevel || {})[msg.level] || json.channel;
+        }, ['items', 'ctx']);
         await got_1.default
             .post(webhookUrl, {
             json,
             responseType: 'text',
+            timeout: 90000,
         })
             .catch(err => {
             // ignore (unless throwOnError is set)

package/dist/slack/slack.service.model.d.ts

@@ -1,5 +1,5 @@
-import { StringMap } from '@naturalcycles/js-lib';
-import { DebugLogLevel, InspectAnyOptions } from '..';
+import { CommonLogger, StringMap } from '@naturalcycles/js-lib';
+import { InspectAnyOptions } from '..';
 /**
  * Properties that exists both in SlackApiBody (as per Slack API) and SlackMessage (our abstraction).
  */
@@ -36,7 +36,6 @@ export interface SlackMessage<CTX = any> extends SlackMessageProps {
      * Optional "context object", to be used by `messagePrefixHook`.
      */
     ctx?: CTX;
-    level?: DebugLogLevel;
     /**
      * Keys-values will be rendered as MessageAttachment with Fields
      */
@@ -46,21 +45,11 @@ export interface SlackMessage<CTX = any> extends SlackMessageProps {
      */
     mentions?: string[];
     /**
-     * @default false
      * By default it ignores possible errors from slack
-     */
-    throwOnError?: boolean;
-    /**
+     *
      * @default false
-     * Skips logging message
-     */
-    noLog?: boolean;
-    /**
-     * Defaults to:
-     * includeErrorData: true
-     * includeErrorStack: true
      */
-    inspectOptions?: InspectAnyOptions;
+    throwOnError?: boolean;
 }
 export interface SlackAttachmentField {
     title: string;
@@ -96,15 +85,21 @@ export interface SlackServiceCfg<CTX = any> {
      */
     webhookUrl?: string;
     defaults?: Partial<SlackMessage>;
-    /**
-     * Override channel when msg.level is set.
-     * key: DebugLogLevel
-     * value: channel name to send message to
-     */
-    channelByLevel?: StringMap;
     /**
      * Function to return an array of "prefix tokens" (will be joined by ': ').
      * Allows to skip (filter out) the message by returning `null`.
      */
     messagePrefixHook: SlackMessagePrefixHook<CTX>;
+    /**
+     * By default SlackService logs every message to console.log
+     * Pass another logger if needed.
+     * Pass `noopLogger` to suppress logging completely.
+     */
+    logger: CommonLogger;
+    /**
+     * Defaults to:
+     * includeErrorData: true
+     * includeErrorStack: true
+     */
+    inspectOptions: InspectAnyOptions;
 }

package/dist/stream/transform/transformLogProgress.js

@@ -47,7 +47,7 @@ function transformLogProgress(opt = {}) {
         const mem = process.memoryUsage();
         const now = Date.now();
         const batchedProgress = progress * batchSize;
-        const lastRPS = (processedLastSecond * batchedProgress) / ((now - lastSecondStarted) / 1000) || 0;
+        const lastRPS = (processedLastSecond * batchSize) / ((now - lastSecondStarted) / 1000) || 0;
         const rpsTotal = Math.round(batchedProgress / ((now - started) / 1000)) || 0;
         lastSecondStarted = now;
         processedLastSecond = 0;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@naturalcycles/nodejs-lib",
-  "version": "12.50.0",
+  "version": "12.51.0",
   "scripts": {
     "prepare": "husky install",
     "docs-serve": "vuepress dev docs",

package/src/security/secret.util.ts

@@ -1,13 +1,10 @@
 import * as fs from 'fs'
 import { StringMap } from '@naturalcycles/js-lib'
-import { base64ToString, Debug } from '..'
+import { base64ToString } from '..'
 import { decryptRandomIVBuffer } from './crypto.util'
 
 let loaded = false
 
-// it's wrapped to be able to pipe console.* to Stackdriver
-const getLog = () => Debug('nc:nodejs-lib:secret')
-
 const secretMap: StringMap = {}
 
 /**
@@ -29,7 +26,7 @@ export function loadSecretsFromEnv(): void {
     })
 
   loaded = true
-  getLog()(
+  console.log(
     `${Object.keys(secrets).length} secret(s) loaded from process.env: ${Object.keys(secrets).join(
       ', ',
     )}`,
@@ -69,7 +66,7 @@ export function loadSecretsFromJsonFile(filePath: string, SECRET_ENCRYPTION_KEY?
   Object.entries(secrets).forEach(([k, v]) => (secretMap[k.toUpperCase()] = v))
 
   loaded = true
-  getLog()(
+  console.log(
     `${Object.keys(secrets).length} secret(s) loaded from ${filePath}: ${Object.keys(secrets)
       .map(s => s.toUpperCase())
       .join(', ')}`,
@@ -105,7 +102,7 @@ export function getSecretMap(): StringMap {
 export function setSecretMap(map: StringMap): void {
   Object.keys(secretMap).forEach(k => delete secretMap[k])
   Object.entries(map).forEach(([k, v]) => (secretMap[k.toUpperCase()] = v))
-  getLog()(
+  console.log(
     `setSecretMap set ${Object.keys(secretMap).length} secret(s): ${Object.keys(map)
       .map(s => s.toUpperCase())
       .join(', ')}`,

package/src/slack/slack.service.model.ts

@@ -1,5 +1,5 @@
-import { StringMap } from '@naturalcycles/js-lib'
-import { DebugLogLevel, InspectAnyOptions } from '..'
+import { CommonLogger, StringMap } from '@naturalcycles/js-lib'
+import { InspectAnyOptions } from '..'
 
 /**
  * Properties that exists both in SlackApiBody (as per Slack API) and SlackMessage (our abstraction).
@@ -44,8 +44,6 @@ export interface SlackMessage<CTX = any> extends SlackMessageProps {
    */
   ctx?: CTX
 
-  level?: DebugLogLevel
-
   /**
    * Keys-values will be rendered as MessageAttachment with Fields
    */
@@ -57,23 +55,11 @@ export interface SlackMessage<CTX = any> extends SlackMessageProps {
   mentions?: string[]
 
   /**
-   * @default false
    * By default it ignores possible errors from slack
-   */
-  throwOnError?: boolean
-
-  /**
+   *
    * @default false
-   * Skips logging message
-   */
-  noLog?: boolean
-
-  /**
-   * Defaults to:
-   * includeErrorData: true
-   * includeErrorStack: true
    */
-  inspectOptions?: InspectAnyOptions
+  throwOnError?: boolean
 }
 
 export interface SlackAttachmentField {
@@ -120,16 +106,23 @@ export interface SlackServiceCfg<CTX = any> {
 
   defaults?: Partial<SlackMessage>
 
-  /**
-   * Override channel when msg.level is set.
-   * key: DebugLogLevel
-   * value: channel name to send message to
-   */
-  channelByLevel?: StringMap
-
   /**
    * Function to return an array of "prefix tokens" (will be joined by ': ').
    * Allows to skip (filter out) the message by returning `null`.
    */
   messagePrefixHook: SlackMessagePrefixHook<CTX>
+
+  /**
+   * By default SlackService logs every message to console.log
+   * Pass another logger if needed.
+   * Pass `noopLogger` to suppress logging completely.
+   */
+  logger: CommonLogger
+
+  /**
+   * Defaults to:
+   * includeErrorData: true
+   * includeErrorStack: true
+   */
+  inspectOptions: InspectAnyOptions
 }

package/src/slack/slack.service.ts

@@ -1,7 +1,7 @@
-import { StringMap } from '@naturalcycles/js-lib'
+import { _omit, StringMap } from '@naturalcycles/js-lib'
 import { dayjs } from '@naturalcycles/time-lib'
 import got from 'got'
-import { Debug, DebugLogLevel, inspectAny, InspectAnyOptions } from '..'
+import { inspectAny, InspectAnyOptions } from '..'
 import {
   SlackApiBody,
   SlackAttachmentField,
@@ -11,12 +11,12 @@ import {
 
 const GAE = !!process.env['GAE_INSTANCE']
 
-const DEFAULTS = (): SlackMessage => ({
+const DEFAULTS: SlackMessage = {
   username: 'bot',
   channel: '#log',
   icon_emoji: ':spider_web:',
   items: 'no text',
-})
+}
 
 const INSPECT_OPT: InspectAnyOptions = {
   colors: false,
@@ -24,8 +24,6 @@ const INSPECT_OPT: InspectAnyOptions = {
   includeErrorStack: true,
 }
 
-const log = Debug('nc:nodejs-lib:slack')
-
 /**
  * Has 2 main methods:
  *
@@ -42,7 +40,12 @@ export class SlackService<CTX = any> {
   constructor(cfg: Partial<SlackServiceCfg<CTX>>) {
     this.cfg = {
       messagePrefixHook: slackDefaultMessagePrefixHook,
+      logger: console,
       ...cfg,
+      inspectOptions: {
+        ...INSPECT_OPT,
+        ...cfg.inspectOptions,
+      },
     }
   }
 
@@ -58,47 +61,34 @@ export class SlackService<CTX = any> {
     })
   }
 
-  async send(msg: SlackMessage<CTX> | string, ctx?: CTX): Promise<void> {
-    const { webhookUrl, messagePrefixHook } = this.cfg
+  async send(input: SlackMessage<CTX> | string, ctx?: CTX): Promise<void> {
+    const { webhookUrl, messagePrefixHook, inspectOptions } = this.cfg
 
     // If String is passed as first argument - just transform it to a full SlackMessage
-    if (typeof msg === 'string') {
-      msg = {
-        items: msg,
-      }
-    }
+    const msg = typeof input === 'string' ? { items: input } : input
 
     if (ctx !== undefined) {
       Object.assign(msg, { ctx })
     }
 
-    if (!msg.noLog) {
-      log[msg.level || DebugLogLevel.info](
-        ...[msg.items, msg.kv, msg.attachments, msg.mentions].filter(Boolean),
-      )
-    }
+    this.cfg.logger.log(...[msg.items, msg.kv, msg.attachments, msg.mentions].filter(Boolean))
 
     if (!webhookUrl) return
 
     // Transform msg.kv into msg.attachments
     if (msg.kv) {
-      msg.attachments = [...(msg.attachments || []), { fields: this.kvToFields(msg.kv) }]
+      ;(msg.attachments ||= []).push({ fields: this.kvToFields(msg.kv) })
 
       delete msg.kv // to not pass it all the way to Slack Api
     }
 
     let text: string
 
-    const inspectOpt = {
-      ...INSPECT_OPT,
-      ...msg.inspectOptions,
-    }
-
     // Array has a special treatment here
     if (Array.isArray(msg.items)) {
-      text = msg.items.map(t => inspectAny(t, inspectOpt)).join('\n')
+      text = msg.items.map(t => inspectAny(t, inspectOptions)).join('\n')
     } else {
-      text = inspectAny(msg.items, inspectOpt)
+      text = inspectAny(msg.items, inspectOptions)
     }
 
     // Wrap in markdown-text-block if it's anything but plain String
@@ -113,29 +103,26 @@ export class SlackService<CTX = any> {
     const prefix = await messagePrefixHook(msg)
     if (prefix === null) return // filtered out!
 
-    const json: SlackApiBody = {
-      ...DEFAULTS(),
-      ...this.cfg.defaults,
-      ...msg,
-      // Text with Prefix
-      text: [prefix.join(': '), text].filter(Boolean).join('\n'),
-    }
-
-    // they're not needed in the json payload
-    delete json['items']
-    delete json['ctx']
-    delete json['noLog']
-
-    json.channel = (this.cfg.channelByLevel || {})[msg.level!] || json.channel
+    const json: SlackApiBody = _omit(
+      {
+        ...DEFAULTS,
+        ...this.cfg.defaults,
+        ...msg,
+        // Text with Prefix
+        text: [prefix.join(': '), text].filter(Boolean).join('\n'),
+      },
+      ['items', 'ctx'],
+    )
 
     await got
       .post(webhookUrl, {
         json,
         responseType: 'text',
+        timeout: 90_000,
       })
       .catch(err => {
         // ignore (unless throwOnError is set)
-        if ((msg as SlackMessage).throwOnError) throw err
+        if (msg.throwOnError) throw err
       })
   }
 

package/src/stream/transform/transformLogProgress.ts

@@ -164,8 +164,7 @@ export function transformLogProgress<IN = any>(
 
     const now = Date.now()
     const batchedProgress = progress * batchSize
-    const lastRPS =
-      (processedLastSecond * batchedProgress) / ((now - lastSecondStarted) / 1000) || 0
+    const lastRPS = (processedLastSecond * batchSize) / ((now - lastSecondStarted) / 1000) || 0
     const rpsTotal = Math.round(batchedProgress / ((now - started) / 1000)) || 0
     lastSecondStarted = now
     processedLastSecond = 0