@naturalcycles/nodejs-lib 12.58.0 → 12.62.0

package/dist/util/zip.util.js

@@ -1,52 +1,57 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.gunzipToString = exports.unzipToString = exports.gzipString = exports.zipString = exports.gunzipBuffer = exports.unzipBuffer = exports.gzipBuffer = exports.zipBuffer = void 0;
+exports.gunzipToString = exports.gzipString = exports.gunzipBuffer = exports.gzipBuffer = exports.inflateToString = exports.deflateString = exports.inflateBuffer = exports.deflateBuffer = void 0;
 const util_1 = require("util");
 const zlib = require("zlib");
 const deflate = (0, util_1.promisify)(zlib.deflate.bind(zlib));
 const inflate = (0, util_1.promisify)(zlib.inflate.bind(zlib));
 const gzip = (0, util_1.promisify)(zlib.gzip.bind(zlib));
 const gunzip = (0, util_1.promisify)(zlib.gunzip.bind(zlib));
-// string > zip
+// string > compressed buffer
 /**
- * zipBuffer uses `deflate`.
- * It's 9 bytes shorter than gzip.
+ * deflateBuffer uses `deflate`.
+ * It's 9 bytes shorter than `gzip`.
  */
-async function zipBuffer(buf, options = {}) {
+async function deflateBuffer(buf, options = {}) {
     return await deflate(buf, options);
 }
-exports.zipBuffer = zipBuffer;
+exports.deflateBuffer = deflateBuffer;
+async function inflateBuffer(buf, options = {}) {
+    return await inflate(buf, options);
+}
+exports.inflateBuffer = inflateBuffer;
+/**
+ * deflateString uses `deflate`.
+ * It's 9 bytes shorter than `gzip`.
+ */
+async function deflateString(s, options) {
+    return await deflateBuffer(Buffer.from(s), options);
+}
+exports.deflateString = deflateString;
+async function inflateToString(buf, options) {
+    return (await inflateBuffer(buf, options)).toString();
+}
+exports.inflateToString = inflateToString;
 /**
  * gzipBuffer uses `gzip`
- * It's 9 bytes longer than deflate.
+ * It's 9 bytes longer than `deflate`.
  */
 async function gzipBuffer(buf, options = {}) {
     return await gzip(buf, options);
 }
 exports.gzipBuffer = gzipBuffer;
-// zip > buffer
-async function unzipBuffer(buf, options = {}) {
-    return await inflate(buf, options);
-}
-exports.unzipBuffer = unzipBuffer;
 async function gunzipBuffer(buf, options = {}) {
     return await gunzip(buf, options);
 }
 exports.gunzipBuffer = gunzipBuffer;
-// convenience
-async function zipString(s, options) {
-    return await zipBuffer(Buffer.from(s), options);
-}
-exports.zipString = zipString;
+/**
+ * gzipString uses `gzip`.
+ * It's 9 bytes longer than `deflate`.
+ */
 async function gzipString(s, options) {
     return await gzipBuffer(Buffer.from(s), options);
 }
 exports.gzipString = gzipString;
-// convenience
-async function unzipToString(buf, options) {
-    return (await unzipBuffer(buf, options)).toString();
-}
-exports.unzipToString = unzipToString;
 async function gunzipToString(buf, options) {
     return (await gunzipBuffer(buf, options)).toString();
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@naturalcycles/nodejs-lib",
-  "version": "12.58.0",
+  "version": "12.62.0",
   "scripts": {
     "prepare": "husky install",
     "docs-serve": "vuepress dev docs",

package/src/got/getGot.ts

@@ -1,8 +1,13 @@
 import { URL } from 'url'
 import { _since } from '@naturalcycles/js-lib'
-import got, { AfterResponseHook, BeforeErrorHook, BeforeRequestHook, Got, HTTPError } from 'got'
+import got, {
+  AfterResponseHook,
+  BeforeErrorHook,
+  BeforeRequestHook,
+  BeforeRetryHook,
+  Got,
+} from 'got'
 import { inspectAny } from '..'
-import { dimGrey, grey, red, yellow } from '../colors'
 import { GetGotOptions, GotRequestContext } from './got.model'
 
 /**
@@ -15,12 +20,33 @@ import { GetGotOptions, GotRequestContext } from './got.model'
 export function getGot(opt: GetGotOptions = {}): Got {
   opt.logger ||= console
 
+  if (opt.debug) {
+    opt.logStart = opt.logFinished = opt.logResponse = true
+  }
+
   return got.extend({
     // Most-important is to set to anything non-empty (so, requests don't "hang" by default).
     // Should be long enough to handle for slow responses from scaled cloud APIs in times of spikes
     // Ideally should be LESS than default Request timeout in backend-lib (so, it has a chance to error
     // before server times out with 503).
-    timeout: 90_000,
+    //
+    // UPD 2021-11-27
+    // There are 2 types/strategies for requests:
+    // 1. Optimized to get result no matter what. E.g in Cron jobs, where otherwise there'll be a job failure
+    // 2. Part of the Backend request, where we better retry quickly and fail on timeout before Backend aborts it with "503 Request timeout"
+    //
+    // Here it's hard to set the default timeout right for both use-cases.
+    // So, if it's important, you should override it according to your use-cases:
+    // - set it longer for Type 1 (e.g 120 seconds)
+    // - set it shorter for Type 2 (e.g 10/20 seconds)
+    // Please beware of default Retry strategy of Got:
+    // by default it will retry 2 times (after first try)
+    // First delay between tries will be ~1 second, then ~2 seconds
+    // Each retry it'll wait up to `timeout` (so, up to 60 seconds by default).
+    // So, for 3 tries it multiplies your timeout by 3 (+3 seconds between the tries).
+    // So, e.g 60 seconds timeout with 2 retries becomes up to 183 seconds.
+    // Which definitely doesn't fit into default "RequestTimeout"
+    timeout: 60_000,
     ...opt,
     hooks: {
       ...opt.hooks,
@@ -34,6 +60,11 @@ export function getGot(opt: GetGotOptions = {}): Got {
         // User hooks go AFTER
         ...(opt.hooks?.beforeRequest || []),
       ],
+      beforeRetry: [
+        gotBeforeRetryHook(opt),
+        // User hooks go AFTER
+        ...(opt.hooks?.beforeRetry || []),
+      ],
       afterResponse: [
         ...(opt.hooks?.afterResponse || []),
         // User hooks go BEFORE
@@ -71,22 +102,38 @@ function gotErrorHook(opt: GetGotOptions = {}): BeforeErrorHook {
   const { maxResponseLength = 10_000 } = opt
 
   return err => {
-    if (err instanceof HTTPError) {
-      const { statusCode } = err.response
-      const { method, url, prefixUrl } = err.options
-      const shortUrl = getShortUrl(opt, url, prefixUrl)
-      // const { started } = context as GotRequestContext
-
-      const body = inspectAny(err.response.body, {
-        maxLen: maxResponseLength,
-        colors: false,
-      })
-
-      // timings are not part of err.message to allow automatic error grouping in Sentry
-      err.message = [[statusCode, method, shortUrl].filter(Boolean).join(' '), body]
+    const statusCode = err.response?.statusCode || 0
+    const { method, url, prefixUrl } = err.options
+    const shortUrl = getShortUrl(opt, url, prefixUrl)
+    const { started, retryCount } = (err.request?.options.context || {}) as GotRequestContext
+
+    const body = err.response?.body
+      ? inspectAny(err.response.body, {
+          maxLen: maxResponseLength,
+          colors: false,
+        })
+      : err.message
+
+    // We don't include Response/Body/Message in the log, because it's included in the Error thrown from here
+    opt.logger!.log(
+      [
+        ' <<',
+        statusCode,
+        method,
+        shortUrl,
+        retryCount && `(retry ${retryCount})`,
+        'error',
+        started && 'in ' + _since(started),
+      ]
         .filter(Boolean)
-        .join('\n')
-    }
+        .join(' '),
+    )
+
+    // timings are not part of err.message to allow automatic error grouping in Sentry
+    // Colors are not used, because there's high chance that this Error will be propagated all the way to the Frontend
+    err.message = [[statusCode, method, shortUrl].filter(Boolean).join(' '), body]
+      .filter(Boolean)
+      .join('\n')
 
     return err
   }
@@ -100,28 +147,76 @@ function gotBeforeRequestHook(opt: GetGotOptions): BeforeRequestHook {
     } as GotRequestContext
 
     if (opt.logStart) {
+      const { retryCount } = options.context as GotRequestContext
       const shortUrl = getShortUrl(opt, options.url, options.prefixUrl)
-      opt.logger!.log([dimGrey(' >>'), dimGrey(options.method), grey(shortUrl)].join(' '))
+      opt.logger!.log(
+        [' >>', options.method, shortUrl, retryCount && `(retry ${retryCount})`].join(' '),
+      )
     }
   }
 }
 
+// Here we log always, because it's similar to ErrorHook - we always log errors
+// Because Retries are always result of some Error
+function gotBeforeRetryHook(opt: GetGotOptions): BeforeRetryHook {
+  const { maxResponseLength = 10_000 } = opt
+
+  return (options, err, retryCount) => {
+    // opt.logger!.log('beforeRetry', retryCount)
+    const statusCode = err?.response?.statusCode || 0
+    const { method, url, prefixUrl } = options
+    const shortUrl = getShortUrl(opt, url, prefixUrl)
+    const { started } = options.context as GotRequestContext
+    Object.assign(options.context, { retryCount })
+
+    const body = err?.response?.body
+      ? inspectAny(err.response.body, {
+          maxLen: maxResponseLength,
+          colors: false,
+        })
+      : err?.message
+
+    // We don't include Response/Body/Message in the log, because it's included in the Error thrown from here
+    opt.logger!.warn(
+      [
+        [
+          ' <<',
+          statusCode,
+          method,
+          shortUrl,
+          retryCount && retryCount > 1 ? `(retry ${retryCount - 1})` : '(first try)',
+          'error',
+          started && 'in ' + _since(started),
+        ]
+          .filter(Boolean)
+          .join(' '),
+        body,
+      ]
+        .filter(Boolean)
+        .join('\n'),
+    )
+  }
+}
+
+// AfterResponseHook is never called on Error
+// So, coloredHttpCode(resp.statusCode) is probably useless
 function gotAfterResponseHook(opt: GetGotOptions = {}): AfterResponseHook {
   return resp => {
     const success = resp.statusCode >= 200 && resp.statusCode < 400
 
     if (opt.logFinished) {
-      const { started } = resp.request.options.context as GotRequestContext
+      const { started, retryCount } = resp.request.options.context as GotRequestContext
       const { url, prefixUrl, method } = resp.request.options
       const shortUrl = getShortUrl(opt, url, prefixUrl)
 
       opt.logger!.log(
         [
-          dimGrey(' <<'),
-          coloredHttpCode(resp.statusCode),
-          dimGrey(method),
-          grey(shortUrl),
-          started && dimGrey('in ' + _since(started)),
+          ' <<',
+          resp.statusCode,
+          method,
+          shortUrl,
+          retryCount && `(retry ${retryCount - 1})`,
+          started && 'in ' + _since(started),
         ]
           .filter(Boolean)
           .join(' '),
@@ -138,12 +233,6 @@ function gotAfterResponseHook(opt: GetGotOptions = {}): AfterResponseHook {
   }
 }
 
-function coloredHttpCode(statusCode: number): string {
-  if (statusCode < 400) return dimGrey(statusCode) // default
-  if (statusCode < 500) return yellow(statusCode)
-  return red(statusCode)
-}
-
 function getShortUrl(opt: GetGotOptions, url: URL, prefixUrl?: string): string {
   let shortUrl = url.toString()
 

package/src/got/got.model.ts

@@ -2,6 +2,12 @@ import { AnyObject, CommonLogger } from '@naturalcycles/js-lib'
 import type { Options } from 'got'
 
 export interface GetGotOptions extends Options {
+  /**
+   * Set to `true` to enable all possible debug logging.
+   * Not safe in prod (as it logs Responses), but great to use during development.
+   */
+  debug?: boolean
+
   /**
    * @default false
    */
@@ -51,4 +57,6 @@ export interface GotRequestContext extends AnyObject {
    * Millisecond-timestamp of when the request was started. To be able to count "time spent".
    */
   started: number
+
+  retryCount?: number
 }

package/src/index.ts

@@ -1,44 +1,18 @@
 import Ajv from 'ajv'
-import { HTTPError, TimeoutError } from 'got'
+import { RequestError, TimeoutError } from 'got'
 import type { AfterResponseHook, BeforeErrorHook, BeforeRequestHook, Got } from 'got'
 import { AnySchema, ValidationErrorItem } from 'joi'
 import { _chunkBuffer } from './buffer/buffer.util'
 import { tableDiff, TableDiffOptions } from './diff/tableDiff'
-import { getGot } from './got/getGot'
+export * from './got/getGot'
 import { GetGotOptions } from './got/got.model'
-import { memoryUsage, memoryUsageFull, processSharedUtil } from './infra/process.util'
+export * from './infra/process.util'
 import { Debug, IDebug, IDebugger } from './log/debug'
-import {
-  base64ToBuffer,
-  base64ToString,
-  bufferToBase64,
-  hash,
-  md5,
-  hashAsBuffer,
-  md5AsBuffer,
-  stringToBase64,
-} from './security/hash.util'
-import {
-  ALPHABET_ALPHANUMERIC,
-  ALPHABET_ALPHANUMERIC_LOWERCASE,
-  ALPHABET_ALPHANUMERIC_UPPERCASE,
-  ALPHABET_LOWERCASE,
-  ALPHABET_NUMBER,
-  ALPHABET_UPPERCASE,
-  stringId,
-  stringIdAsync,
-  stringIdUnsafe,
-} from './security/id.util'
-import {
-  getSecretMap,
-  loadSecretsFromEnv,
-  loadSecretsFromJsonFile,
-  removeSecretsFromEnv,
-  secret,
-  secretOptional,
-  setSecretMap,
-} from './security/secret.util'
-import { hasColors } from './colors/colors'
+export * from './security/hash.util'
+export * from './security/id.util'
+export * from './security/secret.util'
+export * from './colors/colors'
+export * from './log/log.util'
 import { slackDefaultMessagePrefixHook, SlackService } from './slack/slack.service'
 import {
   SlackApiBody,
@@ -48,9 +22,9 @@ import {
   SlackServiceCfg,
 } from './slack/slack.service.model'
 import { NDJsonStats } from './stream/ndjson/ndjson.model'
-import { ndJsonFileRead } from './stream/ndjson/ndJsonFileRead'
-import { ndJsonFileWrite } from './stream/ndjson/ndJsonFileWrite'
-import { ndjsonMap } from './stream/ndjson/ndjsonMap'
+export * from './stream/ndjson/ndJsonFileRead'
+export * from './stream/ndjson/ndJsonFileWrite'
+export * from './stream/ndjson/ndjsonMap'
 import {
   ndjsonStreamForEach,
   NDJSONStreamForEachOptions,
@@ -70,34 +44,31 @@ import {
   TransformJsonParseOptions,
 } from './stream/ndjson/transformJsonParse'
 import { transformToNDJson, TransformToNDJsonOptions } from './stream/ndjson/transformToNDJson'
-import { _pipeline } from './stream/pipeline/pipeline'
-import { readableCreate, readableFrom } from './stream/readable/readableCreate'
-import { readableForEach, readableForEachSync } from './stream/readable/readableForEach'
-import { readableFromArray } from './stream/readable/readableFromArray'
-import { readableMap } from './stream/readable/readableMap'
-import { readableMapToArray } from './stream/readable/readableMapToArray'
-import { readableToArray } from './stream/readable/readableToArray'
+export * from './stream/pipeline/pipeline'
+export * from './stream/readable/readableCreate'
+export * from './stream/readable/readableForEach'
+export * from './stream/readable/readableFromArray'
+export * from './stream/readable/readableMap'
+export * from './stream/readable/readableMapToArray'
+export * from './stream/readable/readableToArray'
 import {
   ReadableTyped,
   TransformOptions,
   TransformTyped,
   WritableTyped,
 } from './stream/stream.model'
-import { transformBuffer } from './stream/transform/transformBuffer'
-import { transformFilter, transformFilterSync } from './stream/transform/transformFilter'
-import { transformLimit } from './stream/transform/transformLimit'
-import {
-  transformLogProgress,
-  TransformLogProgressOptions,
-} from './stream/transform/transformLogProgress'
+export * from './stream/transform/transformBuffer'
+export * from './stream/transform/transformFilter'
+export * from './stream/transform/transformLimit'
+export * from './stream/transform/transformLogProgress'
 import { transformMap, TransformMapOptions } from './stream/transform/transformMap'
-import { transformMapSimple } from './stream/transform/transformMapSimple'
-import { transformNoOp } from './stream/transform/transformNoOp'
+export * from './stream/transform/transformMapSimple'
+export * from './stream/transform/transformNoOp'
 import { transformMapSync, TransformMapSyncOptions } from './stream/transform/transformMapSync'
-import { transformSplit } from './stream/transform/transformSplit'
-import { transformTap } from './stream/transform/transformTap'
-import { transformToArray } from './stream/transform/transformToArray'
-import { transformToString } from './stream/transform/transformToString'
+export * from './stream/transform/transformSplit'
+export * from './stream/transform/transformTap'
+export * from './stream/transform/transformToArray'
+export * from './stream/transform/transformToString'
 import { BaseWorkerClass, WorkerClassInterface } from './stream/transform/worker/baseWorkerClass'
 import {
   transformMultiThreaded,
@@ -105,26 +76,17 @@ import {
 } from './stream/transform/worker/transformMultiThreaded'
 import { WorkerInput, WorkerOutput } from './stream/transform/worker/transformMultiThreaded.model'
 export * from './stream/writable/writableForEach'
-import { writableFork } from './stream/writable/writableFork'
-import { writablePushToArray } from './stream/writable/writablePushToArray'
-import { writableVoid } from './stream/writable/writableVoid'
+export * from './stream/writable/writableFork'
+export * from './stream/writable/writablePushToArray'
+export * from './stream/writable/writableVoid'
 import { inspectAny, InspectAnyOptions, inspectAnyStringifyFn } from './string/inspectAny'
-import { requireEnvKeys, requireFileToExist } from './util/env.util'
+export * from './util/env.util'
 import { LRUMemoCache } from './util/lruMemoCache'
-import {
-  gunzipBuffer,
-  gunzipToString,
-  gzipBuffer,
-  gzipString,
-  unzipBuffer,
-  unzipToString,
-  zipBuffer,
-  zipString,
-} from './util/zip.util'
+export * from './util/zip.util'
 import { readAjvSchemas, readJsonSchemas } from './validation/ajv/ajv.util'
 import { AjvSchema, AjvSchemaCfg, AjvValidationOptions } from './validation/ajv/ajvSchema'
 import { AjvValidationError, AjvValidationErrorData } from './validation/ajv/ajvValidationError'
-import { getAjv } from './validation/ajv/getAjv'
+export * from './validation/ajv/getAjv'
 import { ExtendedJoi, Joi } from './validation/joi/joi.extensions'
 import {
   AnySchemaTyped,
@@ -135,34 +97,7 @@ import {
   SchemaTyped,
   StringSchemaTyped,
 } from './validation/joi/joi.model'
-import {
-  anyObjectSchema,
-  anySchema,
-  arraySchema,
-  oneOfSchema,
-  binarySchema,
-  booleanDefaultToFalseSchema,
-  booleanSchema,
-  dateStringSchema,
-  emailSchema,
-  baseDBEntitySchema,
-  savedDBEntitySchema,
-  idSchema,
-  integerSchema,
-  ipAddressSchema,
-  numberSchema,
-  objectSchema,
-  percentageSchema,
-  semVerSchema,
-  SEM_VER_PATTERN,
-  slugSchema,
-  stringSchema,
-  unixTimestampSchema,
-  urlSchema,
-  userAgentSchema,
-  utcOffsetSchema,
-  verSchema,
-} from './validation/joi/joi.shared.schemas'
+export * from './validation/joi/joi.shared.schemas'
 import { JoiValidationError, JoiValidationErrorData } from './validation/joi/joi.validation.error'
 import {
   convert,
@@ -207,7 +142,6 @@ export type {
   TransformMapSyncOptions,
   NDJSONStreamForEachOptions,
   TransformOptions,
-  TransformLogProgressOptions,
   TransformMultiThreadedOptions,
   WorkerClassInterface,
   WorkerInput,
@@ -233,86 +167,10 @@ export {
   undefinedIfInvalid,
   convert,
   Joi,
-  booleanSchema,
-  booleanDefaultToFalseSchema,
-  stringSchema,
-  numberSchema,
-  integerSchema,
-  percentageSchema,
-  dateStringSchema,
-  arraySchema,
-  binarySchema,
-  objectSchema,
-  oneOfSchema,
-  anySchema,
-  anyObjectSchema,
-  baseDBEntitySchema,
-  savedDBEntitySchema,
-  idSchema,
-  unixTimestampSchema,
-  verSchema,
-  emailSchema,
-  SEM_VER_PATTERN,
-  semVerSchema,
-  userAgentSchema,
-  utcOffsetSchema,
-  ipAddressSchema,
-  slugSchema,
-  urlSchema,
-  processSharedUtil,
-  zipBuffer,
-  gzipBuffer,
-  unzipBuffer,
-  gunzipBuffer,
-  zipString,
-  gzipString,
-  unzipToString,
-  gunzipToString,
-  requireEnvKeys,
-  requireFileToExist,
   LRUMemoCache,
-  stringId,
-  stringIdAsync,
-  stringIdUnsafe,
-  ALPHABET_NUMBER,
-  ALPHABET_LOWERCASE,
-  ALPHABET_UPPERCASE,
-  ALPHABET_ALPHANUMERIC_LOWERCASE,
-  ALPHABET_ALPHANUMERIC_UPPERCASE,
-  ALPHABET_ALPHANUMERIC,
-  md5,
-  hash,
-  hashAsBuffer,
-  md5AsBuffer,
-  stringToBase64,
-  base64ToString,
-  bufferToBase64,
-  base64ToBuffer,
   Debug,
-  getSecretMap,
-  setSecretMap,
-  loadSecretsFromEnv,
-  loadSecretsFromJsonFile,
-  removeSecretsFromEnv,
-  secret,
-  secretOptional,
-  memoryUsage,
-  memoryUsageFull,
   SlackService,
   slackDefaultMessagePrefixHook,
-  readableCreate,
-  readableFrom,
-  readableFromArray,
-  readableToArray,
-  readableForEach,
-  readableForEachSync,
-  readableMap,
-  readableMapToArray,
-  _pipeline,
-  transformBuffer,
-  ndjsonMap,
-  ndJsonFileRead,
-  ndJsonFileWrite,
   ndjsonStreamForEach,
   pipelineFromNDJsonFile,
   pipelineToNDJsonFile,
@@ -321,37 +179,21 @@ export {
   transformJsonParse,
   bufferReviver,
   transformToNDJson,
-  transformFilter,
-  transformFilterSync,
   transformMap,
   transformMapSync,
-  transformMapSimple,
-  transformNoOp,
-  writablePushToArray,
-  transformSplit,
-  transformToString,
-  transformToArray,
-  transformTap,
-  transformLogProgress,
-  transformLimit,
-  writableVoid,
-  writableFork,
   transformMultiThreaded,
   BaseWorkerClass,
   tableDiff,
   inspectAny,
   inspectAnyStringifyFn,
-  getGot,
-  HTTPError,
+  RequestError,
   TimeoutError,
   _chunkBuffer,
   Ajv,
-  getAjv,
   AjvSchema,
   AjvValidationError,
   readJsonSchemas,
   readAjvSchemas,
-  hasColors,
   sanitizeHTML,
   runScript,
 }

package/src/log/log.util.ts

@@ -0,0 +1,9 @@
+import { commonLoggerCreate } from '@naturalcycles/js-lib'
+import { inspectAny } from '../index'
+
+/**
+ * CommonLogger that logs to process.stdout directly (bypassing console.log).
+ */
+export const stdoutLogger = commonLoggerCreate((_level, args) => {
+  process.stdout.write(args.map(a => inspectAny(a)).join(' ') + '\n')
+})