@naturalcycles/nodejs-lib 12.59.0 → 12.62.1

package/src/got/getGot.ts

@@ -1,8 +1,13 @@
 import { URL } from 'url'
 import { _since } from '@naturalcycles/js-lib'
-import got, { AfterResponseHook, BeforeErrorHook, BeforeRequestHook, Got, HTTPError } from 'got'
+import got, {
+  AfterResponseHook,
+  BeforeErrorHook,
+  BeforeRequestHook,
+  BeforeRetryHook,
+  Got,
+} from 'got'
 import { inspectAny } from '..'
-import { dimGrey, grey, red, yellow } from '../colors'
 import { GetGotOptions, GotRequestContext } from './got.model'
 
 /**
@@ -15,12 +20,33 @@ import { GetGotOptions, GotRequestContext } from './got.model'
 export function getGot(opt: GetGotOptions = {}): Got {
   opt.logger ||= console
 
+  if (opt.debug) {
+    opt.logStart = opt.logFinished = opt.logResponse = true
+  }
+
   return got.extend({
     // Most-important is to set to anything non-empty (so, requests don't "hang" by default).
     // Should be long enough to handle for slow responses from scaled cloud APIs in times of spikes
     // Ideally should be LESS than default Request timeout in backend-lib (so, it has a chance to error
     // before server times out with 503).
-    timeout: 90_000,
+    //
+    // UPD 2021-11-27
+    // There are 2 types/strategies for requests:
+    // 1. Optimized to get result no matter what. E.g in Cron jobs, where otherwise there'll be a job failure
+    // 2. Part of the Backend request, where we better retry quickly and fail on timeout before Backend aborts it with "503 Request timeout"
+    //
+    // Here it's hard to set the default timeout right for both use-cases.
+    // So, if it's important, you should override it according to your use-cases:
+    // - set it longer for Type 1 (e.g 120 seconds)
+    // - set it shorter for Type 2 (e.g 10/20 seconds)
+    // Please beware of default Retry strategy of Got:
+    // by default it will retry 2 times (after first try)
+    // First delay between tries will be ~1 second, then ~2 seconds
+    // Each retry it'll wait up to `timeout` (so, up to 60 seconds by default).
+    // So, for 3 tries it multiplies your timeout by 3 (+3 seconds between the tries).
+    // So, e.g 60 seconds timeout with 2 retries becomes up to 183 seconds.
+    // Which definitely doesn't fit into default "RequestTimeout"
+    timeout: 60_000,
     ...opt,
     hooks: {
       ...opt.hooks,
@@ -34,6 +60,11 @@ export function getGot(opt: GetGotOptions = {}): Got {
         // User hooks go AFTER
         ...(opt.hooks?.beforeRequest || []),
       ],
+      beforeRetry: [
+        gotBeforeRetryHook(opt),
+        // User hooks go AFTER
+        ...(opt.hooks?.beforeRetry || []),
+      ],
       afterResponse: [
         ...(opt.hooks?.afterResponse || []),
         // User hooks go BEFORE
@@ -71,22 +102,38 @@ function gotErrorHook(opt: GetGotOptions = {}): BeforeErrorHook {
   const { maxResponseLength = 10_000 } = opt
 
   return err => {
-    if (err instanceof HTTPError) {
-      const { statusCode } = err.response
-      const { method, url, prefixUrl } = err.options
-      const shortUrl = getShortUrl(opt, url, prefixUrl)
-      // const { started } = context as GotRequestContext
-
-      const body = inspectAny(err.response.body, {
-        maxLen: maxResponseLength,
-        colors: false,
-      })
-
-      // timings are not part of err.message to allow automatic error grouping in Sentry
-      err.message = [[statusCode, method, shortUrl].filter(Boolean).join(' '), body]
+    const statusCode = err.response?.statusCode || 0
+    const { method, url, prefixUrl } = err.options
+    const shortUrl = getShortUrl(opt, url, prefixUrl)
+    const { started, retryCount } = (err.request?.options.context || {}) as GotRequestContext
+
+    const body = err.response?.body
+      ? inspectAny(err.response.body, {
+          maxLen: maxResponseLength,
+          colors: false,
+        })
+      : err.message
+
+    // We don't include Response/Body/Message in the log, because it's included in the Error thrown from here
+    opt.logger!.log(
+      [
+        ' <<',
+        statusCode,
+        method,
+        shortUrl,
+        retryCount && `(retry ${retryCount})`,
+        'error',
+        started && 'in ' + _since(started),
+      ]
         .filter(Boolean)
-        .join('\n')
-    }
+        .join(' '),
+    )
+
+    // timings are not part of err.message to allow automatic error grouping in Sentry
+    // Colors are not used, because there's high chance that this Error will be propagated all the way to the Frontend
+    err.message = [[statusCode, method, shortUrl].filter(Boolean).join(' '), body]
+      .filter(Boolean)
+      .join('\n')
 
     return err
   }
@@ -100,28 +147,76 @@ function gotBeforeRequestHook(opt: GetGotOptions): BeforeRequestHook {
     } as GotRequestContext
 
     if (opt.logStart) {
+      const { retryCount } = options.context as GotRequestContext
       const shortUrl = getShortUrl(opt, options.url, options.prefixUrl)
-      opt.logger!.log([dimGrey(' >>'), dimGrey(options.method), grey(shortUrl)].join(' '))
+      opt.logger!.log(
+        [' >>', options.method, shortUrl, retryCount && `(retry ${retryCount})`].join(' '),
+      )
     }
   }
 }
 
+// Here we log always, because it's similar to ErrorHook - we always log errors
+// Because Retries are always result of some Error
+function gotBeforeRetryHook(opt: GetGotOptions): BeforeRetryHook {
+  const { maxResponseLength = 10_000 } = opt
+
+  return (options, err, retryCount) => {
+    // opt.logger!.log('beforeRetry', retryCount)
+    const statusCode = err?.response?.statusCode || 0
+    const { method, url, prefixUrl } = options
+    const shortUrl = getShortUrl(opt, url, prefixUrl)
+    const { started } = options.context as GotRequestContext
+    Object.assign(options.context, { retryCount })
+
+    const body = err?.response?.body
+      ? inspectAny(err.response.body, {
+          maxLen: maxResponseLength,
+          colors: false,
+        })
+      : err?.message
+
+    // We don't include Response/Body/Message in the log, because it's included in the Error thrown from here
+    opt.logger!.warn(
+      [
+        [
+          ' <<',
+          statusCode,
+          method,
+          shortUrl,
+          retryCount && retryCount > 1 ? `(retry ${retryCount - 1})` : '(first try)',
+          'error',
+          started && 'in ' + _since(started),
+        ]
+          .filter(Boolean)
+          .join(' '),
+        body,
+      ]
+        .filter(Boolean)
+        .join('\n'),
+    )
+  }
+}
+
+// AfterResponseHook is never called on Error
+// So, coloredHttpCode(resp.statusCode) is probably useless
 function gotAfterResponseHook(opt: GetGotOptions = {}): AfterResponseHook {
   return resp => {
     const success = resp.statusCode >= 200 && resp.statusCode < 400
 
     if (opt.logFinished) {
-      const { started } = resp.request.options.context as GotRequestContext
+      const { started, retryCount } = resp.request.options.context as GotRequestContext
       const { url, prefixUrl, method } = resp.request.options
       const shortUrl = getShortUrl(opt, url, prefixUrl)
 
       opt.logger!.log(
         [
-          dimGrey(' <<'),
-          coloredHttpCode(resp.statusCode),
-          dimGrey(method),
-          grey(shortUrl),
-          started && dimGrey('in ' + _since(started)),
+          ' <<',
+          resp.statusCode,
+          method,
+          shortUrl,
+          retryCount && `(retry ${retryCount - 1})`,
+          started && 'in ' + _since(started),
         ]
           .filter(Boolean)
           .join(' '),
@@ -138,12 +233,6 @@ function gotAfterResponseHook(opt: GetGotOptions = {}): AfterResponseHook {
   }
 }
 
-function coloredHttpCode(statusCode: number): string {
-  if (statusCode < 400) return dimGrey(statusCode) // default
-  if (statusCode < 500) return yellow(statusCode)
-  return red(statusCode)
-}
-
 function getShortUrl(opt: GetGotOptions, url: URL, prefixUrl?: string): string {
   let shortUrl = url.toString()
 

package/src/got/got.model.ts

@@ -2,6 +2,12 @@ import { AnyObject, CommonLogger } from '@naturalcycles/js-lib'
 import type { Options } from 'got'
 
 export interface GetGotOptions extends Options {
+  /**
+   * Set to `true` to enable all possible debug logging.
+   * Not safe in prod (as it logs Responses), but great to use during development.
+   */
+  debug?: boolean
+
   /**
    * @default false
    */
@@ -51,4 +57,6 @@ export interface GotRequestContext extends AnyObject {
    * Millisecond-timestamp of when the request was started. To be able to count "time spent".
    */
   started: number
+
+  retryCount?: number
 }

package/src/index.ts

@@ -1,5 +1,5 @@
 import Ajv from 'ajv'
-import { HTTPError, TimeoutError } from 'got'
+import { RequestError, TimeoutError } from 'got'
 import type { AfterResponseHook, BeforeErrorHook, BeforeRequestHook, Got } from 'got'
 import { AnySchema, ValidationErrorItem } from 'joi'
 import { _chunkBuffer } from './buffer/buffer.util'
@@ -11,7 +11,7 @@ import { Debug, IDebug, IDebugger } from './log/debug'
 export * from './security/hash.util'
 export * from './security/id.util'
 export * from './security/secret.util'
-import { hasColors } from './colors/colors'
+export * from './colors/colors'
 export * from './log/log.util'
 import { slackDefaultMessagePrefixHook, SlackService } from './slack/slack.service'
 import {
@@ -22,9 +22,9 @@ import {
   SlackServiceCfg,
 } from './slack/slack.service.model'
 import { NDJsonStats } from './stream/ndjson/ndjson.model'
-import { ndJsonFileRead } from './stream/ndjson/ndJsonFileRead'
-import { ndJsonFileWrite } from './stream/ndjson/ndJsonFileWrite'
-import { ndjsonMap } from './stream/ndjson/ndjsonMap'
+export * from './stream/ndjson/ndJsonFileRead'
+export * from './stream/ndjson/ndJsonFileWrite'
+export * from './stream/ndjson/ndjsonMap'
 import {
   ndjsonStreamForEach,
   NDJSONStreamForEachOptions,
@@ -45,12 +45,12 @@ import {
 } from './stream/ndjson/transformJsonParse'
 import { transformToNDJson, TransformToNDJsonOptions } from './stream/ndjson/transformToNDJson'
 export * from './stream/pipeline/pipeline'
-import { readableCreate, readableFrom } from './stream/readable/readableCreate'
-import { readableForEach, readableForEachSync } from './stream/readable/readableForEach'
-import { readableFromArray } from './stream/readable/readableFromArray'
-import { readableMap } from './stream/readable/readableMap'
-import { readableMapToArray } from './stream/readable/readableMapToArray'
-import { readableToArray } from './stream/readable/readableToArray'
+export * from './stream/readable/readableCreate'
+export * from './stream/readable/readableForEach'
+export * from './stream/readable/readableFromArray'
+export * from './stream/readable/readableMap'
+export * from './stream/readable/readableMapToArray'
+export * from './stream/readable/readableToArray'
 import {
   ReadableTyped,
   TransformOptions,
@@ -62,13 +62,13 @@ export * from './stream/transform/transformFilter'
 export * from './stream/transform/transformLimit'
 export * from './stream/transform/transformLogProgress'
 import { transformMap, TransformMapOptions } from './stream/transform/transformMap'
-import { transformMapSimple } from './stream/transform/transformMapSimple'
-import { transformNoOp } from './stream/transform/transformNoOp'
+export * from './stream/transform/transformMapSimple'
+export * from './stream/transform/transformNoOp'
 import { transformMapSync, TransformMapSyncOptions } from './stream/transform/transformMapSync'
-import { transformSplit } from './stream/transform/transformSplit'
-import { transformTap } from './stream/transform/transformTap'
-import { transformToArray } from './stream/transform/transformToArray'
-import { transformToString } from './stream/transform/transformToString'
+export * from './stream/transform/transformSplit'
+export * from './stream/transform/transformTap'
+export * from './stream/transform/transformToArray'
+export * from './stream/transform/transformToString'
 import { BaseWorkerClass, WorkerClassInterface } from './stream/transform/worker/baseWorkerClass'
 import {
   transformMultiThreaded,
@@ -86,7 +86,7 @@ export * from './util/zip.util'
 import { readAjvSchemas, readJsonSchemas } from './validation/ajv/ajv.util'
 import { AjvSchema, AjvSchemaCfg, AjvValidationOptions } from './validation/ajv/ajvSchema'
 import { AjvValidationError, AjvValidationErrorData } from './validation/ajv/ajvValidationError'
-import { getAjv } from './validation/ajv/getAjv'
+export * from './validation/ajv/getAjv'
 import { ExtendedJoi, Joi } from './validation/joi/joi.extensions'
 import {
   AnySchemaTyped,
@@ -171,17 +171,6 @@ export {
   Debug,
   SlackService,
   slackDefaultMessagePrefixHook,
-  readableCreate,
-  readableFrom,
-  readableFromArray,
-  readableToArray,
-  readableForEach,
-  readableForEachSync,
-  readableMap,
-  readableMapToArray,
-  ndjsonMap,
-  ndJsonFileRead,
-  ndJsonFileWrite,
   ndjsonStreamForEach,
   pipelineFromNDJsonFile,
   pipelineToNDJsonFile,
@@ -192,27 +181,19 @@ export {
   transformToNDJson,
   transformMap,
   transformMapSync,
-  transformMapSimple,
-  transformNoOp,
-  transformSplit,
-  transformToString,
-  transformToArray,
-  transformTap,
   transformMultiThreaded,
   BaseWorkerClass,
   tableDiff,
   inspectAny,
   inspectAnyStringifyFn,
-  HTTPError,
+  RequestError,
   TimeoutError,
   _chunkBuffer,
   Ajv,
-  getAjv,
   AjvSchema,
   AjvValidationError,
   readJsonSchemas,
   readAjvSchemas,
-  hasColors,
   sanitizeHTML,
   runScript,
 }

package/src/stream/ndjson/transformJsonParse.ts

@@ -33,17 +33,17 @@ export function transformJsonParse<OUT = any>(
   const { strict = true, reviver } = opt
 
   return new Transform({
-    objectMode: false,
+    writableObjectMode: false,
     readableObjectMode: true,
-    transform(chunk: string, _encoding, cb) {
+    transform(chunk: string, _, cb) {
       try {
         const data = JSON.parse(chunk, reviver)
         cb(null, data)
       } catch (err) {
-        // console.error(err)
         if (strict) {
           cb(err as Error) // emit error
         } else {
+          console.error(err)
           cb() // emit no error, but no result neither
         }
       }

package/src/stream/ndjson/transformToNDJson.ts

@@ -37,9 +37,9 @@ export function transformToNDJson<IN = any>(
   const { strict = true, separator = '\n', sortObjects = false, useFlatstr = false } = opt
 
   return new Transform({
-    objectMode: true,
+    writableObjectMode: true,
     readableObjectMode: false,
-    transform(chunk: IN, _encoding, cb) {
+    transform(chunk: IN, _, cb) {
       try {
         if (sortObjects) {
           chunk = _sortObjectDeep(chunk as any)

package/src/stream/sizeStack.ts

@@ -0,0 +1,56 @@
+import { _hb, CommonLogger, NumberStack } from '@naturalcycles/js-lib'
+import { yellow } from '../colors'
+import { gzipBuffer } from '../util/zip.util'
+
+export class SizeStack extends NumberStack {
+  constructor(public name: string, size: number) {
+    super(size)
+  }
+
+  total = 0
+
+  override push(item: any): this {
+    this.total += item
+    return super.push(item)
+  }
+
+  getStats(): string {
+    // const pcs = this.percentiles([50, 90])
+
+    return [
+      '  ' + this.name,
+      'avg',
+      yellow(_hb(this.avg())),
+      // 'p50',
+      // yellow(_hb(pcs[50])),
+      // 'p90',
+      // yellow(_hb(pcs[90])),
+      'total',
+      yellow(_hb(this.total)),
+    ].join(' ')
+  }
+
+  static async countItem(
+    item: any,
+    logger: CommonLogger,
+    sizes?: SizeStack,
+    sizesZipped?: SizeStack,
+  ): Promise<void> {
+    if (!sizes) return
+
+    // try-catch, because we don't want to fail the pipeline on logProgress
+    try {
+      const buf = Buffer.from(JSON.stringify(item))
+      sizes.push(buf.byteLength)
+
+      if (sizesZipped) {
+        const { byteLength } = await gzipBuffer(buf)
+        sizesZipped.push(byteLength)
+      }
+    } catch (err) {
+      logger.warn(
+        `transformLogProgress failed to JSON.stringify the chunk: ${(err as Error).message}`,
+      )
+    }
+  }
+}

package/src/stream/transform/transformBuffer.ts

@@ -18,7 +18,7 @@ export function transformBuffer<IN = any>(opt: TransformBufferOptions): Transfor
   return new Transform({
     objectMode: true,
     ...opt,
-    transform(chunk, _encoding, cb) {
+    transform(chunk, _, cb) {
       buf.push(chunk)
 
       if (buf.length >= batchSize) {

package/src/stream/transform/transformFilter.ts

@@ -1,32 +1,18 @@
 import { Transform } from 'stream'
 import { AsyncPredicate, Predicate } from '@naturalcycles/js-lib'
 import { TransformOptions, TransformTyped } from '../stream.model'
+import { transformMap, TransformMapOptions } from './transformMap'
 
 /**
- * Note, that currently it's NOT concurrent! (concurrency = 1)
- * So, it's recommended to use transformMap instead, that is both concurrent and has
- * filtering feature by default.
+ * Just a convenience wrapper around `transformMap` that has built-in predicate filtering support.
  */
 export function transformFilter<IN = any>(
   predicate: AsyncPredicate<IN>,
-  opt: TransformOptions = {},
+  opt: TransformMapOptions = {},
 ): TransformTyped<IN, IN> {
-  let index = 0
-
-  return new Transform({
-    objectMode: true,
+  return transformMap(v => v, {
+    predicate,
     ...opt,
-    async transform(chunk: IN, _encoding, cb) {
-      try {
-        if (await predicate(chunk, index++)) {
-          cb(null, chunk) // pass through
-        } else {
-          cb() // signal that we've finished processing, but emit no output here
-        }
-      } catch (err) {
-        cb(err as Error)
-      }
-    },
   })
 }
 
@@ -42,7 +28,7 @@ export function transformFilterSync<IN = any>(
   return new Transform({
     objectMode: true,
     ...opt,
-    async transform(chunk: IN, _encoding, cb) {
+    transform(chunk: IN, _, cb) {
       try {
         if (predicate(chunk, index++)) {
           cb(null, chunk) // pass through

package/src/stream/transform/transformLogProgress.ts

@@ -4,6 +4,7 @@ import { SimpleMovingAverage, _mb, _since, AnyObject, CommonLogger } from '@natu
 import { dayjs } from '@naturalcycles/time-lib'
 import { boldWhite, dimGrey, white, yellow } from '../../colors'
 import { hasColors } from '../../colors/colors'
+import { SizeStack } from '../sizeStack'
 import { TransformOptions, TransformTyped } from '../stream.model'
 
 export interface TransformLogProgressOptions<IN = any> extends TransformOptions {
@@ -103,6 +104,41 @@ export interface TransformLogProgressOptions<IN = any> extends TransformOptions
    * Defaults to 1.
    */
   batchSize?: number
+
+  /**
+   * Experimental logging of item (shunk) sizes, when json-stringified.
+   *
+   * Defaults to false.
+   *
+   * @experimental
+   */
+  logSizes?: boolean
+
+  /**
+   * How many last item sizes to keep in a buffer, to calculate stats (p50, p90, avg, etc).
+   * Defaults to 100_000.
+   * Cannot be Infinity.
+   */
+  logSizesBuffer?: number
+
+  /**
+   * Works in addition to `logSizes`. Adds "zipped sizes".
+   *
+   * @experimental
+   */
+  logZippedSizes?: boolean
+}
+
+interface LogItem extends AnyObject {
+  heapUsed?: number
+  heapTotal?: number
+  rss?: number
+  peakRSS?: number
+  rssMinusHeap?: number
+  external?: number
+  arrayBuffers?: number
+  rps10?: number
+  rpsTotal?: number
 }
 
 const inspectOpt: InspectOptions = {
@@ -124,6 +160,9 @@ export function transformLogProgress<IN = any>(
     peakRSS: logPeakRSS = true,
     logRPS = true,
     logEvery = 1000,
+    logSizes = false,
+    logSizesBuffer = 100_000,
+    logZippedSizes = false,
     batchSize = 1,
     extra,
     logger = console,
@@ -138,15 +177,23 @@ export function transformLogProgress<IN = any>(
   let progress = 0
   let peakRSS = 0
 
+  const sizes = logSizes ? new SizeStack('json', logSizesBuffer) : undefined
+  const sizesZipped = logZippedSizes ? new SizeStack('json.gz', logSizesBuffer) : undefined
+
   logStats() // initial
 
   return new Transform({
     objectMode: true,
     ...opt,
-    transform(chunk: IN, _encoding, cb) {
+    transform(chunk: IN, _, cb) {
       progress++
       processedLastSecond++
 
+      if (sizes) {
+        // Check it, cause gzipping might be delayed here..
+        void SizeStack.countItem(chunk, logger, sizes, sizesZipped)
+      }
+
       if (logProgress && progress % logEvery === 0) {
         logStats(chunk, false, progress % logEvery10 === 0)
       }
@@ -175,28 +222,30 @@ export function transformLogProgress<IN = any>(
     const rps10 = Math.round(sma.push(lastRPS))
     if (mem.rss > peakRSS) peakRSS = mem.rss
 
-    logger.log(
-      inspect(
-        {
-          [final ? `${metric}_final` : metric]: batchedProgress,
-          ...(extra ? extra(chunk, progress) : {}),
-          ...(logHeapUsed ? { heapUsed: _mb(mem.heapUsed) } : {}),
-          ...(logHeapTotal ? { heapTotal: _mb(mem.heapTotal) } : {}),
-          ...(logRss ? { rss: _mb(mem.rss) } : {}),
-          ...(logPeakRSS ? { peakRSS: _mb(peakRSS) } : {}),
-          ...(opt.rssMinusHeap ? { rssMinusHeap: _mb(mem.rss - mem.heapTotal) } : {}),
-          ...(opt.external ? { external: _mb(mem.external) } : {}),
-          ...(opt.arrayBuffers ? { arrayBuffers: _mb(mem.arrayBuffers || 0) } : {}),
-          ...(logRPS
-            ? {
-                rps10,
-                rpsTotal,
-              }
-            : {}),
-        },
-        inspectOpt,
-      ),
-    )
+    const o: LogItem = {
+      [final ? `${metric}_final` : metric]: batchedProgress,
+    }
+
+    if (extra) Object.assign(o, extra(chunk, progress))
+    if (logHeapUsed) o.heapUsed = _mb(mem.heapUsed)
+    if (logHeapTotal) o.heapTotal = _mb(mem.heapTotal)
+    if (logRss) o.rss = _mb(mem.rss)
+    if (logPeakRSS) o.peakRSS = _mb(peakRSS)
+    if (opt.rssMinusHeap) o.rssMinusHeap = _mb(mem.rss - mem.heapTotal)
+    if (opt.external) o.external = _mb(mem.external)
+    if (opt.arrayBuffers) o.arrayBuffers = _mb(mem.arrayBuffers || 0)
+
+    if (logRPS) Object.assign(o, { rps10, rpsTotal })
+
+    logger.log(inspect(o, inspectOpt))
+
+    if (sizes?.items.length) {
+      logger.log(sizes.getStats())
+
+      if (sizesZipped?.items.length) {
+        logger.log(sizesZipped.getStats())
+      }
+    }
 
     if (tenx) {
       let perHour: number | string =