@naturalcycles/js-lib 14.144.1 → 14.146.0

package/src/http/fetcher.ts

@@ -15,7 +15,6 @@ import {
 import { pDelay } from '../promise/pDelay'
 import { _jsonParse, _jsonParseIfPossible } from '../string/json.util'
 import { _since } from '../time/time.util'
-import { UnixTimestampNumber } from '../types'
 import type {
   FetcherAfterResponseHook,
   FetcherBeforeRequestHook,
@@ -54,7 +53,10 @@ export class Fetcher {
       const m = method.toLowerCase()
 
       // mode=void
-      ;(this as any)[`${m}Void`] = async (url: string, opt?: FetcherOptions): Promise<void> => {
+      ;(this as any)[`${m}Void`] = async (
+        url: string,
+        opt?: FetcherOptions<void>,
+      ): Promise<void> => {
         return await this.fetch<void>(url, {
           method,
           mode: 'void',
@@ -63,7 +65,10 @@ export class Fetcher {
       }
 
       if (method === 'HEAD') return // mode=text
-      ;(this as any)[`${m}Text`] = async (url: string, opt?: FetcherOptions): Promise<string> => {
+      ;(this as any)[`${m}Text`] = async (
+        url: string,
+        opt?: FetcherOptions<string>,
+      ): Promise<string> => {
         return await this.fetch<string>(url, {
           method,
           mode: 'text',
@@ -72,7 +77,7 @@ export class Fetcher {
       }
 
       // Default mode=json, but overridable
-      ;(this as any)[m] = async <T = unknown>(url: string, opt?: FetcherOptions): Promise<T> => {
+      ;(this as any)[m] = async <T = unknown>(url: string, opt?: FetcherOptions<T>): Promise<T> => {
         return await this.fetch<T>(url, {
           method,
           mode: 'json',
@@ -108,28 +113,45 @@ export class Fetcher {
 
   // These methods are generated dynamically in the constructor
   // These default methods use mode=json
-  get!: <T = unknown>(url: string, opt?: FetcherOptions) => Promise<T>
-  post!: <T = unknown>(url: string, opt?: FetcherOptions) => Promise<T>
-  put!: <T = unknown>(url: string, opt?: FetcherOptions) => Promise<T>
-  patch!: <T = unknown>(url: string, opt?: FetcherOptions) => Promise<T>
-  delete!: <T = unknown>(url: string, opt?: FetcherOptions) => Promise<T>
+  get!: <T = unknown>(url: string, opt?: FetcherOptions<T>) => Promise<T>
+  post!: <T = unknown>(url: string, opt?: FetcherOptions<T>) => Promise<T>
+  put!: <T = unknown>(url: string, opt?: FetcherOptions<T>) => Promise<T>
+  patch!: <T = unknown>(url: string, opt?: FetcherOptions<T>) => Promise<T>
+  delete!: <T = unknown>(url: string, opt?: FetcherOptions<T>) => Promise<T>
 
   // mode=text
-  getText!: (url: string, opt?: FetcherOptions) => Promise<string>
-  postText!: (url: string, opt?: FetcherOptions) => Promise<string>
-  putText!: (url: string, opt?: FetcherOptions) => Promise<string>
-  patchText!: (url: string, opt?: FetcherOptions) => Promise<string>
-  deleteText!: (url: string, opt?: FetcherOptions) => Promise<string>
+  getText!: (url: string, opt?: FetcherOptions<string>) => Promise<string>
+  postText!: (url: string, opt?: FetcherOptions<string>) => Promise<string>
+  putText!: (url: string, opt?: FetcherOptions<string>) => Promise<string>
+  patchText!: (url: string, opt?: FetcherOptions<string>) => Promise<string>
+  deleteText!: (url: string, opt?: FetcherOptions<string>) => Promise<string>
 
   // mode=void (no body fetching/parsing)
-  getVoid!: (url: string, opt?: FetcherOptions) => Promise<void>
-  postVoid!: (url: string, opt?: FetcherOptions) => Promise<void>
-  putVoid!: (url: string, opt?: FetcherOptions) => Promise<void>
-  patchVoid!: (url: string, opt?: FetcherOptions) => Promise<void>
-  deleteVoid!: (url: string, opt?: FetcherOptions) => Promise<void>
-  headVoid!: (url: string, opt?: FetcherOptions) => Promise<void>
-
-  async fetch<T = unknown>(url: string, opt?: FetcherOptions): Promise<T> {
+  getVoid!: (url: string, opt?: FetcherOptions<void>) => Promise<void>
+  postVoid!: (url: string, opt?: FetcherOptions<void>) => Promise<void>
+  putVoid!: (url: string, opt?: FetcherOptions<void>) => Promise<void>
+  patchVoid!: (url: string, opt?: FetcherOptions<void>) => Promise<void>
+  deleteVoid!: (url: string, opt?: FetcherOptions<void>) => Promise<void>
+  headVoid!: (url: string, opt?: FetcherOptions<void>) => Promise<void>
+
+  // mode=readableStream
+  /**
+   * Returns raw fetchResponse.body, which is a ReadableStream<Uint8Array>
+   *
+   * More on streams and Node interop:
+   * https://css-tricks.com/web-streams-everywhere-and-fetch-for-node-js/
+   */
+  async getReadableStream(
+    url: string,
+    opt?: FetcherOptions<ReadableStream<Uint8Array>>,
+  ): Promise<ReadableStream<Uint8Array>> {
+    return await this.fetch(url, {
+      mode: 'readableStream',
+      ...opt,
+    })
+  }
+
+  async fetch<T = unknown>(url: string, opt?: FetcherOptions<T>): Promise<T> {
     const res = await this.doFetch<T>(url, opt)
     if (res.err) {
       if (res.req.throwHttpErrors) throw res.err
@@ -145,11 +167,14 @@ export class Fetcher {
    */
   async doFetch<T = unknown>(
     url: string,
-    rawOpt: FetcherOptions = {},
+    opt: FetcherOptions<T> = {},
   ): Promise<FetcherResponse<T>> {
-    const { logger } = this.cfg
+    const req = this.normalizeOptions(url, opt)
+    return await this.doFetchRequest<T>(req)
+  }
 
-    const req = this.normalizeOptions(url, rawOpt)
+  async doFetchRequest<T = unknown>(req: FetcherRequest<T>): Promise<FetcherResponse<T>> {
+    const { logger } = this.cfg
     const {
       timeoutSeconds,
       init: { method },
@@ -165,7 +190,7 @@ export class Fetcher {
       }, timeoutSeconds * 1000) as any as number
     }
 
-    for await (const hook of this.cfg.hooks.beforeRequest || []) {
+    for (const hook of this.cfg.hooks.beforeRequest || []) {
       await hook(req)
     }
 
@@ -185,7 +210,7 @@ export class Fetcher {
     } as FetcherResponse<any>
 
     while (!res.retryStatus.retryStopped) {
-      const started = Date.now()
+      req.started = Date.now()
 
       if (this.cfg.logRequest) {
         const { retryAttempt } = res.retryStatus
@@ -212,27 +237,29 @@ export class Fetcher {
       res.statusFamily = this.getStatusFamily(res)
 
       if (res.fetchResponse?.ok) {
-        await this.onOkResponse(
-          res as FetcherResponse<T> & { fetchResponse: Response },
-          started,
-          timeout,
-        )
+        await this.onOkResponse(res as FetcherResponse<T> & { fetchResponse: Response }, timeout)
       } else {
         // !res.ok
-        await this.onNotOkResponse(res, started, timeout)
+        await this.onNotOkResponse(res, timeout)
       }
     }
 
-    for await (const hook of this.cfg.hooks.afterResponse || []) {
+    for (const hook of this.cfg.hooks.afterResponse || []) {
       await hook(res)
     }
 
+    if (req.paginate && res.ok) {
+      const proceed = await req.paginate(req, res)
+      if (proceed) {
+        return await this.doFetchRequest(req)
+      }
+    }
+
     return res
   }
 
   private async onOkResponse(
     res: FetcherResponse<any> & { fetchResponse: Response },
-    started: UnixTimestampNumber,
     timeout?: number,
   ): Promise<void> {
     const { req } = res
@@ -259,7 +286,7 @@ export class Fetcher {
             res.err = _anyToError(err)
             res.ok = false
 
-            return await this.onNotOkResponse(res, started, timeout)
+            return await this.onNotOkResponse(res, timeout)
           }
         } else {
           // Body had a '' (empty string)
@@ -276,6 +303,14 @@ export class Fetcher {
       res.body = res.fetchResponse.body ? await res.fetchResponse.arrayBuffer() : {}
     } else if (mode === 'blob') {
       res.body = res.fetchResponse.body ? await res.fetchResponse.blob() : {}
+    } else if (mode === 'readableStream') {
+      res.body = res.fetchResponse.body
+
+      if (res.body === null) {
+        res.err = new Error(`fetchResponse.body is null`)
+        res.ok = false
+        return await this.onNotOkResponse(res, timeout)
+      }
     }
 
     clearTimeout(timeout)
@@ -291,7 +326,7 @@ export class Fetcher {
           res.fetchResponse.status,
           res.signature,
           retryAttempt && `try#${retryAttempt + 1}/${req.retry.count + 1}`,
-          _since(started),
+          _since(res.req.started),
         ]
           .filter(Boolean)
           .join(' '),
@@ -310,11 +345,7 @@ export class Fetcher {
     return await globalThis.fetch(url, init)
   }
 
-  private async onNotOkResponse(
-    res: FetcherResponse,
-    started: UnixTimestampNumber,
-    timeout?: number,
-  ): Promise<void> {
+  private async onNotOkResponse(res: FetcherResponse, timeout?: number): Promise<void> {
     clearTimeout(timeout)
 
     let cause: ErrorObject | undefined
@@ -345,7 +376,7 @@ export class Fetcher {
         requestBaseUrl: this.cfg.baseUrl || (null as any),
         requestMethod: res.req.init.method,
         requestSignature: res.signature,
-        requestDuration: Date.now() - started,
+        requestDuration: Date.now() - res.req.started,
       }),
       cause,
     )
@@ -360,7 +391,7 @@ export class Fetcher {
       retryStatus.retryStopped = true
     }
 
-    for await (const hook of this.cfg.hooks.beforeRetry || []) {
+    for (const hook of this.cfg.hooks.beforeRetry || []) {
       await hook(res)
     }
 
@@ -372,6 +403,26 @@ export class Fetcher {
 
     if (retryStatus.retryStopped) return
 
+    // Here we know that more retries will be attempted
+    // We don't log "last error", because it will be thrown and logged by consumer,
+    // but we should log all previous errors, otherwise they are lost.
+    // Here is the right place where we know it's not the "last error"
+    if (res.err) {
+      const { retryAttempt } = retryStatus
+      this.cfg.logger.error(
+        [
+          ' <<',
+          res.fetchResponse?.status || 0,
+          res.signature,
+          `try#${retryAttempt + 1}/${count + 1}`,
+          _since(res.req.started),
+        ]
+          .filter(Boolean)
+          .join(' '),
+        res.err.cause || res.err,
+      )
+    }
+
     retryStatus.retryAttempt++
     retryStatus.retryTimeout = _clamp(retryStatus.retryTimeout * timeoutMultiplier, 0, timeoutMax)
 
@@ -477,7 +528,7 @@ export class Fetcher {
     return norm
   }
 
-  private normalizeOptions(url: string, opt: FetcherOptions): FetcherRequest {
+  private normalizeOptions<BODY>(url: string, opt: FetcherOptions<BODY>): FetcherRequest<BODY> {
     const {
       timeoutSeconds,
       throwHttpErrors,
@@ -489,7 +540,8 @@ export class Fetcher {
       jsonReviver,
     } = this.cfg
 
-    const req: FetcherRequest = {
+    const req: FetcherRequest<BODY> = {
+      started: Date.now(),
       mode,
       url,
       timeoutSeconds,

package/src/json-schema/jsonSchema.model.ts

@@ -1,6 +1,5 @@
 import type { AnyObject, StringMap } from '../types'
 
-// eslint-disable-next-line unused-imports/no-unused-vars
 export type JsonSchema<T = unknown> =
   | JsonSchemaAny<T>
   | JsonSchemaOneOf<T>

package/src/object/object.util.ts

@@ -139,7 +139,6 @@ export function _mapKeys<T extends AnyObject>(
   obj: T,
   mapper: ObjectMapper<T, string>,
 ): StringMap<T[keyof T]> {
-  // eslint-disable-next-line unicorn/prefer-object-from-entries
   return Object.entries(obj).reduce((map, [k, v]) => {
     map[mapper(k, v, obj) as keyof T] = v
     return map

package/src/object/sortObjectDeep.ts

@@ -6,7 +6,6 @@ import { _isObject } from '..'
 export function _sortObjectDeep<T>(o: T): T {
   // array
   if (Array.isArray(o)) {
-    // eslint-disable-next-line unicorn/no-array-callback-reference
     return o.map(_sortObjectDeep) as any
   }
 

package/src/promise/pMap.ts

@@ -77,7 +77,7 @@ export async function pMap<IN, OUT>(
   if (concurrency === 1) {
     // Special case for concurrency == 1
 
-    for await (const item of items) {
+    for (const item of items) {
       try {
         const r = await mapper(item, currentIndex++)
         if (r === END) break

package/src/string/leven.ts

@@ -70,7 +70,7 @@ export function _leven(first: string, second: string): number {
     for (index = 0; index < firstLength; index++) {
       temporary2 = bCharacterCode === characterCodeCache[index] ? temporary : temporary + 1
       temporary = array[index]!
-      // eslint-disable-next-line no-multi-assign
+
       result = array[index] =
         temporary > result
           ? temporary2 > result

package/src/string/string.util.ts

@@ -1,5 +1,3 @@
-/* eslint-disable unicorn/prefer-string-slice */
-
 /**
  * Converts the first character of string to upper case and the remaining to lower case.
  */

package/src/vendor/is.ts

@@ -84,7 +84,6 @@ function isPrimitiveTypeName(name: unknown): name is PrimitiveTypeName {
 
 export type TypeName = ObjectTypeName | PrimitiveTypeName
 
-// eslint-disable-next-line @typescript-eslint/ban-types
 function isOfType<T extends Primitive | Function>(type: PrimitiveTypeName | 'function') {
   return (value: unknown): value is T => typeof value === type
 }
@@ -168,7 +167,6 @@ is.number = (value: unknown): value is number => isNumberType(value) && !is.nan(
 
 is.bigint = isOfType<bigint>('bigint')
 
-// eslint-disable-next-line @typescript-eslint/ban-types
 is.function_ = isOfType<Function>('function')
 
 is.null_ = (value: unknown): value is null => value === null
@@ -228,7 +226,6 @@ is.asyncGeneratorFunction = (value: unknown): value is (...args: any[]) => Promi
 is.asyncFunction = <T = unknown>(value: unknown): value is (...args: any[]) => Promise<T> =>
   getObjectType(value) === 'AsyncFunction'
 
-// eslint-disable-next-line no-prototype-builtins, @typescript-eslint/ban-types
 is.boundFunction = (value: unknown): value is Function =>
   is.function_(value) && !value.hasOwnProperty('prototype')
 
@@ -270,7 +267,7 @@ is.urlString = (value: unknown): value is string => {
   }
 
   try {
-    new URL(value) // eslint-disable-line no-new
+    new URL(value)
     return true
   } catch {
     return false
@@ -512,7 +509,7 @@ interface Assert {
   string: (value: unknown) => asserts value is string
   number: (value: unknown) => asserts value is number
   bigint: (value: unknown) => asserts value is bigint
-  // eslint-disable-next-line @typescript-eslint/ban-types
+
   function_: (value: unknown) => asserts value is Function
   null_: (value: unknown) => asserts value is null
   class_: (value: unknown) => asserts value is Class
@@ -536,9 +533,9 @@ interface Assert {
   promise: <T = unknown>(value: unknown) => asserts value is Promise<T>
   generatorFunction: (value: unknown) => asserts value is GeneratorFunction
   asyncGeneratorFunction: (value: unknown) => asserts value is AsyncGeneratorFunction
-  // eslint-disable-next-line @typescript-eslint/ban-types
+
   asyncFunction: (value: unknown) => asserts value is Function
-  // eslint-disable-next-line @typescript-eslint/ban-types
+
   boundFunction: (value: unknown) => asserts value is Function
   regExp: (value: unknown) => asserts value is RegExp
   date: (value: unknown) => asserts value is Date
@@ -620,7 +617,7 @@ export const assert: Assert = {
     assertType(is.number(value), 'number', value),
   bigint: (value: unknown): asserts value is bigint =>
     assertType(is.bigint(value), 'bigint', value),
-  // eslint-disable-next-line @typescript-eslint/ban-types
+
   function_: (value: unknown): asserts value is Function =>
     assertType(is.function_(value), 'Function', value),
   null_: (value: unknown): asserts value is null => assertType(is.null_(value), 'null', value),
@@ -666,10 +663,10 @@ export const assert: Assert = {
     assertType(is.generatorFunction(value), 'GeneratorFunction', value),
   asyncGeneratorFunction: (value: unknown): asserts value is AsyncGeneratorFunction =>
     assertType(is.asyncGeneratorFunction(value), 'AsyncGeneratorFunction', value),
-  // eslint-disable-next-line @typescript-eslint/ban-types
+
   asyncFunction: (value: unknown): asserts value is Function =>
     assertType(is.asyncFunction(value), 'AsyncFunction', value),
-  // eslint-disable-next-line @typescript-eslint/ban-types
+
   boundFunction: (value: unknown): asserts value is Function =>
     assertType(is.boundFunction(value), 'Function', value),
   regExp: (value: unknown): asserts value is RegExp =>