@naturalcycles/js-lib 14.145.0 → 14.146.0

package/dist/http/fetcher.d.ts

@@ -1,5 +1,5 @@
 /// <reference lib="dom" />
-import type { FetcherAfterResponseHook, FetcherBeforeRequestHook, FetcherBeforeRetryHook, FetcherCfg, FetcherNormalizedCfg, FetcherOptions, FetcherResponse } from './fetcher.model';
+import type { FetcherAfterResponseHook, FetcherBeforeRequestHook, FetcherBeforeRetryHook, FetcherCfg, FetcherNormalizedCfg, FetcherOptions, FetcherRequest, FetcherResponse } from './fetcher.model';
 /**
  * Experimental wrapper around Fetch.
  * Works in both Browser and Node, using `globalThis.fetch`.
@@ -14,36 +14,37 @@ export declare class Fetcher {
     onBeforeRetry(hook: FetcherBeforeRetryHook): this;
     cfg: FetcherNormalizedCfg;
     static create(cfg?: FetcherCfg & FetcherOptions): Fetcher;
-    get: <T = unknown>(url: string, opt?: FetcherOptions) => Promise<T>;
-    post: <T = unknown>(url: string, opt?: FetcherOptions) => Promise<T>;
-    put: <T = unknown>(url: string, opt?: FetcherOptions) => Promise<T>;
-    patch: <T = unknown>(url: string, opt?: FetcherOptions) => Promise<T>;
-    delete: <T = unknown>(url: string, opt?: FetcherOptions) => Promise<T>;
-    getText: (url: string, opt?: FetcherOptions) => Promise<string>;
-    postText: (url: string, opt?: FetcherOptions) => Promise<string>;
-    putText: (url: string, opt?: FetcherOptions) => Promise<string>;
-    patchText: (url: string, opt?: FetcherOptions) => Promise<string>;
-    deleteText: (url: string, opt?: FetcherOptions) => Promise<string>;
-    getVoid: (url: string, opt?: FetcherOptions) => Promise<void>;
-    postVoid: (url: string, opt?: FetcherOptions) => Promise<void>;
-    putVoid: (url: string, opt?: FetcherOptions) => Promise<void>;
-    patchVoid: (url: string, opt?: FetcherOptions) => Promise<void>;
-    deleteVoid: (url: string, opt?: FetcherOptions) => Promise<void>;
-    headVoid: (url: string, opt?: FetcherOptions) => Promise<void>;
+    get: <T = unknown>(url: string, opt?: FetcherOptions<T>) => Promise<T>;
+    post: <T = unknown>(url: string, opt?: FetcherOptions<T>) => Promise<T>;
+    put: <T = unknown>(url: string, opt?: FetcherOptions<T>) => Promise<T>;
+    patch: <T = unknown>(url: string, opt?: FetcherOptions<T>) => Promise<T>;
+    delete: <T = unknown>(url: string, opt?: FetcherOptions<T>) => Promise<T>;
+    getText: (url: string, opt?: FetcherOptions<string>) => Promise<string>;
+    postText: (url: string, opt?: FetcherOptions<string>) => Promise<string>;
+    putText: (url: string, opt?: FetcherOptions<string>) => Promise<string>;
+    patchText: (url: string, opt?: FetcherOptions<string>) => Promise<string>;
+    deleteText: (url: string, opt?: FetcherOptions<string>) => Promise<string>;
+    getVoid: (url: string, opt?: FetcherOptions<void>) => Promise<void>;
+    postVoid: (url: string, opt?: FetcherOptions<void>) => Promise<void>;
+    putVoid: (url: string, opt?: FetcherOptions<void>) => Promise<void>;
+    patchVoid: (url: string, opt?: FetcherOptions<void>) => Promise<void>;
+    deleteVoid: (url: string, opt?: FetcherOptions<void>) => Promise<void>;
+    headVoid: (url: string, opt?: FetcherOptions<void>) => Promise<void>;
     /**
      * Returns raw fetchResponse.body, which is a ReadableStream<Uint8Array>
      *
      * More on streams and Node interop:
      * https://css-tricks.com/web-streams-everywhere-and-fetch-for-node-js/
      */
-    getReadableStream(url: string, opt?: FetcherOptions): Promise<ReadableStream<Uint8Array>>;
-    fetch<T = unknown>(url: string, opt?: FetcherOptions): Promise<T>;
+    getReadableStream(url: string, opt?: FetcherOptions<ReadableStream<Uint8Array>>): Promise<ReadableStream<Uint8Array>>;
+    fetch<T = unknown>(url: string, opt?: FetcherOptions<T>): Promise<T>;
     /**
      * Returns FetcherResponse.
      * Never throws, returns `err` property in the response instead.
      * Use this method instead of `throwHttpErrors: false` or try-catching.
      */
-    doFetch<T = unknown>(url: string, rawOpt?: FetcherOptions): Promise<FetcherResponse<T>>;
+    doFetch<T = unknown>(url: string, opt?: FetcherOptions<T>): Promise<FetcherResponse<T>>;
+    doFetchRequest<T = unknown>(req: FetcherRequest<T>): Promise<FetcherResponse<T>>;
     private onOkResponse;
     /**
      * This method exists to be able to easily mock it.

package/dist/http/fetcher.js

@@ -104,9 +104,12 @@ class Fetcher {
      * Never throws, returns `err` property in the response instead.
      * Use this method instead of `throwHttpErrors: false` or try-catching.
      */
-    async doFetch(url, rawOpt = {}) {
+    async doFetch(url, opt = {}) {
+        const req = this.normalizeOptions(url, opt);
+        return await this.doFetchRequest(req);
+    }
+    async doFetchRequest(req) {
         const { logger } = this.cfg;
-        const req = this.normalizeOptions(url, rawOpt);
         const { timeoutSeconds, init: { method }, } = req;
         // setup timeout
         let timeout;
@@ -117,7 +120,7 @@ class Fetcher {
                 abortController.abort(`timeout of ${timeoutSeconds} sec`);
             }, timeoutSeconds * 1000);
         }
-        for await (const hook of this.cfg.hooks.beforeRequest || []) {
+        for (const hook of this.cfg.hooks.beforeRequest || []) {
             await hook(req);
         }
         const isFullUrl = req.url.includes('://');
@@ -134,7 +137,7 @@ class Fetcher {
             signature,
         };
         while (!res.retryStatus.retryStopped) {
-            const started = Date.now();
+            req.started = Date.now();
             if (this.cfg.logRequest) {
                 const { retryAttempt } = res.retryStatus;
                 logger.log([' >>', signature, retryAttempt && `try#${retryAttempt + 1}/${req.retry.count + 1}`]
@@ -157,19 +160,25 @@ class Fetcher {
             }
             res.statusFamily = this.getStatusFamily(res);
             if (res.fetchResponse?.ok) {
-                await this.onOkResponse(res, started, timeout);
+                await this.onOkResponse(res, timeout);
             }
             else {
                 // !res.ok
-                await this.onNotOkResponse(res, started, timeout);
+                await this.onNotOkResponse(res, timeout);
             }
         }
-        for await (const hook of this.cfg.hooks.afterResponse || []) {
+        for (const hook of this.cfg.hooks.afterResponse || []) {
             await hook(res);
         }
+        if (req.paginate && res.ok) {
+            const proceed = await req.paginate(req, res);
+            if (proceed) {
+                return await this.doFetchRequest(req);
+            }
+        }
         return res;
     }
-    async onOkResponse(res, started, timeout) {
+    async onOkResponse(res, timeout) {
         const { req } = res;
         const { mode } = res.req;
         if (mode === 'json') {
@@ -192,7 +201,7 @@ class Fetcher {
                         // } satisfies HttpRequestErrorData)
                         res.err = (0, error_util_1._anyToError)(err);
                         res.ok = false;
-                        return await this.onNotOkResponse(res, started, timeout);
+                        return await this.onNotOkResponse(res, timeout);
                     }
                 }
                 else {
@@ -220,7 +229,7 @@ class Fetcher {
             if (res.body === null) {
                 res.err = new Error(`fetchResponse.body is null`);
                 res.ok = false;
-                return await this.onNotOkResponse(res, started, timeout);
+                return await this.onNotOkResponse(res, timeout);
             }
         }
         clearTimeout(timeout);
@@ -234,7 +243,7 @@ class Fetcher {
                 res.fetchResponse.status,
                 res.signature,
                 retryAttempt && `try#${retryAttempt + 1}/${req.retry.count + 1}`,
-                (0, time_util_1._since)(started),
+                (0, time_util_1._since)(res.req.started),
             ]
                 .filter(Boolean)
                 .join(' '));
@@ -249,7 +258,7 @@ class Fetcher {
     async callNativeFetch(url, init) {
         return await globalThis.fetch(url, init);
     }
-    async onNotOkResponse(res, started, timeout) {
+    async onNotOkResponse(res, timeout) {
         clearTimeout(timeout);
         let cause;
         if (res.err) {
@@ -275,7 +284,7 @@ class Fetcher {
             requestBaseUrl: this.cfg.baseUrl || null,
             requestMethod: res.req.init.method,
             requestSignature: res.signature,
-            requestDuration: Date.now() - started,
+            requestDuration: Date.now() - res.req.started,
         }), cause);
         await this.processRetry(res);
     }
@@ -284,7 +293,7 @@ class Fetcher {
         if (!this.shouldRetry(res)) {
             retryStatus.retryStopped = true;
         }
-        for await (const hook of this.cfg.hooks.beforeRetry || []) {
+        for (const hook of this.cfg.hooks.beforeRetry || []) {
             await hook(res);
         }
         const { count, timeoutMultiplier, timeoutMax } = res.req.retry;
@@ -293,6 +302,22 @@ class Fetcher {
         }
         if (retryStatus.retryStopped)
             return;
+        // Here we know that more retries will be attempted
+        // We don't log "last error", because it will be thrown and logged by consumer,
+        // but we should log all previous errors, otherwise they are lost.
+        // Here is the right place where we know it's not the "last error"
+        if (res.err) {
+            const { retryAttempt } = retryStatus;
+            this.cfg.logger.error([
+                ' <<',
+                res.fetchResponse?.status || 0,
+                res.signature,
+                `try#${retryAttempt + 1}/${count + 1}`,
+                (0, time_util_1._since)(res.req.started),
+            ]
+                .filter(Boolean)
+                .join(' '), res.err.cause || res.err);
+        }
         retryStatus.retryAttempt++;
         retryStatus.retryTimeout = (0, number_util_1._clamp)(retryStatus.retryTimeout * timeoutMultiplier, 0, timeoutMax);
         const noise = Math.random() * 500;
@@ -393,6 +418,7 @@ class Fetcher {
     normalizeOptions(url, opt) {
         const { timeoutSeconds, throwHttpErrors, retryPost, retry4xx, retry5xx, retry, mode, jsonReviver, } = this.cfg;
         const req = {
+            started: Date.now(),
             mode,
             url,
             timeoutSeconds,

package/dist/http/fetcher.model.d.ts

@@ -1,14 +1,14 @@
 import type { CommonLogger } from '../log/commonLogger';
 import type { Promisable } from '../typeFest';
-import type { Reviver } from '../types';
+import type { Reviver, UnixTimestampMillisNumber } from '../types';
 import type { HttpMethod, HttpStatusFamily } from './http.model';
-export interface FetcherNormalizedCfg extends Required<FetcherCfg>, FetcherRequest {
+export interface FetcherNormalizedCfg extends Required<FetcherCfg>, Omit<FetcherRequest, 'started'> {
     logger: CommonLogger;
     searchParams: Record<string, any>;
 }
-export type FetcherBeforeRequestHook = (req: FetcherRequest) => Promisable<void>;
-export type FetcherAfterResponseHook = (res: FetcherResponse) => Promisable<void>;
-export type FetcherBeforeRetryHook = (res: FetcherResponse) => Promisable<void>;
+export type FetcherBeforeRequestHook = <BODY = unknown>(req: FetcherRequest<BODY>) => Promisable<void>;
+export type FetcherAfterResponseHook = <BODY = unknown>(res: FetcherResponse<BODY>) => Promisable<void>;
+export type FetcherBeforeRetryHook = <BODY = unknown>(res: FetcherResponse<BODY>) => Promisable<void>;
 export interface FetcherCfg {
     /**
      * Should **not** contain trailing slash.
@@ -77,7 +77,7 @@ export interface FetcherRetryOptions {
     timeoutMax: number;
     timeoutMultiplier: number;
 }
-export interface FetcherRequest extends Omit<FetcherOptions, 'method' | 'headers' | 'baseUrl'> {
+export interface FetcherRequest<BODY = unknown> extends Omit<FetcherOptions<BODY>, 'method' | 'headers' | 'baseUrl'> {
     url: string;
     init: RequestInitNormalized;
     mode: FetcherMode;
@@ -87,8 +87,9 @@ export interface FetcherRequest extends Omit<FetcherOptions, 'method' | 'headers
     retryPost: boolean;
     retry4xx: boolean;
     retry5xx: boolean;
+    started: UnixTimestampMillisNumber;
 }
-export interface FetcherOptions {
+export interface FetcherOptions<BODY = unknown> {
     method?: HttpMethod;
     baseUrl?: string;
     throwHttpErrors?: boolean;
@@ -134,6 +135,18 @@ export interface FetcherOptions {
      */
     retry5xx?: boolean;
     jsonReviver?: Reviver;
+    /**
+     * Allows to walk over multiple pages of results.
+     * Paginate take a function.
+     * Function has access to FetcherRequest and FetcherResponse
+     * and has to make a decision to continue pagination or not.
+     * Return true to continue, false otherwise.
+     * If continue - it is expected to modify/mutate the FetcherRequest, as it will be used
+     * to request the next page.
+     *
+     * @experimental
+     */
+    paginate?: (req: FetcherRequest<BODY>, res: FetcherSuccessResponse<BODY>) => Promisable<boolean>;
 }
 export type RequestInitNormalized = Omit<RequestInit, 'method' | 'headers'> & {
     method: HttpMethod;

package/dist/promise/pMap.js

@@ -51,7 +51,7 @@ async function pMap(iterable, mapper, opt = {}) {
     // Special cases that are able to preserve async stack traces
     if (concurrency === 1) {
         // Special case for concurrency == 1
-        for await (const item of items) {
+        for (const item of items) {
             try {
                 const r = await mapper(item, currentIndex++);
                 if (r === __1.END)

package/dist-esm/http/fetcher.js

@@ -1,5 +1,4 @@
 /// <reference lib="dom"/>
-import { __asyncValues } from "tslib";
 import { isServerSide } from '../env';
 import { _anyToError, _anyToErrorObject, _errorToErrorObject } from '../error/error.util';
 import { HttpRequestError } from '../error/httpRequestError';
@@ -90,11 +89,13 @@ export class Fetcher {
      * Never throws, returns `err` property in the response instead.
      * Use this method instead of `throwHttpErrors: false` or try-catching.
      */
-    async doFetch(url, rawOpt = {}) {
-        var _a, e_1, _b, _c, _d, e_2, _e, _f;
-        var _g;
+    async doFetch(url, opt = {}) {
+        const req = this.normalizeOptions(url, opt);
+        return await this.doFetchRequest(req);
+    }
+    async doFetchRequest(req) {
+        var _a;
         const { logger } = this.cfg;
-        const req = this.normalizeOptions(url, rawOpt);
         const { timeoutSeconds, init: { method }, } = req;
         // setup timeout
         let timeout;
@@ -105,25 +106,8 @@ export class Fetcher {
                 abortController.abort(`timeout of ${timeoutSeconds} sec`);
             }, timeoutSeconds * 1000);
         }
-        try {
-            for (var _h = true, _j = __asyncValues(this.cfg.hooks.beforeRequest || []), _k; _k = await _j.next(), _a = _k.done, !_a;) {
-                _c = _k.value;
-                _h = false;
-                try {
-                    const hook = _c;
-                    await hook(req);
-                }
-                finally {
-                    _h = true;
-                }
-            }
-        }
-        catch (e_1_1) { e_1 = { error: e_1_1 }; }
-        finally {
-            try {
-                if (!_h && !_a && (_b = _j.return)) await _b.call(_j);
-            }
-            finally { if (e_1) throw e_1.error; }
+        for (const hook of this.cfg.hooks.beforeRequest || []) {
+            await hook(req);
         }
         const isFullUrl = req.url.includes('://');
         const fullUrl = isFullUrl ? new URL(req.url) : undefined;
@@ -139,7 +123,7 @@ export class Fetcher {
             signature,
         };
         while (!res.retryStatus.retryStopped) {
-            const started = Date.now();
+            req.started = Date.now();
             if (this.cfg.logRequest) {
                 const { retryAttempt } = res.retryStatus;
                 logger.log([' >>', signature, retryAttempt && `try#${retryAttempt + 1}/${req.retry.count + 1}`]
@@ -161,37 +145,26 @@ export class Fetcher {
                 res.fetchResponse = undefined;
             }
             res.statusFamily = this.getStatusFamily(res);
-            if ((_g = res.fetchResponse) === null || _g === void 0 ? void 0 : _g.ok) {
-                await this.onOkResponse(res, started, timeout);
+            if ((_a = res.fetchResponse) === null || _a === void 0 ? void 0 : _a.ok) {
+                await this.onOkResponse(res, timeout);
             }
             else {
                 // !res.ok
-                await this.onNotOkResponse(res, started, timeout);
+                await this.onNotOkResponse(res, timeout);
             }
         }
-        try {
-            for (var _l = true, _m = __asyncValues(this.cfg.hooks.afterResponse || []), _o; _o = await _m.next(), _d = _o.done, !_d;) {
-                _f = _o.value;
-                _l = false;
-                try {
-                    const hook = _f;
-                    await hook(res);
-                }
-                finally {
-                    _l = true;
-                }
-            }
+        for (const hook of this.cfg.hooks.afterResponse || []) {
+            await hook(res);
         }
-        catch (e_2_1) { e_2 = { error: e_2_1 }; }
-        finally {
-            try {
-                if (!_l && !_d && (_e = _m.return)) await _e.call(_m);
+        if (req.paginate && res.ok) {
+            const proceed = await req.paginate(req, res);
+            if (proceed) {
+                return await this.doFetchRequest(req);
             }
-            finally { if (e_2) throw e_2.error; }
         }
         return res;
     }
-    async onOkResponse(res, started, timeout) {
+    async onOkResponse(res, timeout) {
         const { req } = res;
         const { mode } = res.req;
         if (mode === 'json') {
@@ -214,7 +187,7 @@ export class Fetcher {
                         // } satisfies HttpRequestErrorData)
                         res.err = _anyToError(err);
                         res.ok = false;
-                        return await this.onNotOkResponse(res, started, timeout);
+                        return await this.onNotOkResponse(res, timeout);
                     }
                 }
                 else {
@@ -242,7 +215,7 @@ export class Fetcher {
             if (res.body === null) {
                 res.err = new Error(`fetchResponse.body is null`);
                 res.ok = false;
-                return await this.onNotOkResponse(res, started, timeout);
+                return await this.onNotOkResponse(res, timeout);
             }
         }
         clearTimeout(timeout);
@@ -256,7 +229,7 @@ export class Fetcher {
                 res.fetchResponse.status,
                 res.signature,
                 retryAttempt && `try#${retryAttempt + 1}/${req.retry.count + 1}`,
-                _since(started),
+                _since(res.req.started),
             ]
                 .filter(Boolean)
                 .join(' '));
@@ -271,7 +244,7 @@ export class Fetcher {
     async callNativeFetch(url, init) {
         return await globalThis.fetch(url, init);
     }
-    async onNotOkResponse(res, started, timeout) {
+    async onNotOkResponse(res, timeout) {
         var _a, _b;
         clearTimeout(timeout);
         let cause;
@@ -298,35 +271,18 @@ export class Fetcher {
             requestBaseUrl: this.cfg.baseUrl || null,
             requestMethod: res.req.init.method,
             requestSignature: res.signature,
-            requestDuration: Date.now() - started,
+            requestDuration: Date.now() - res.req.started,
         }), cause);
         await this.processRetry(res);
     }
     async processRetry(res) {
-        var _a, e_3, _b, _c;
+        var _a;
         const { retryStatus } = res;
         if (!this.shouldRetry(res)) {
             retryStatus.retryStopped = true;
         }
-        try {
-            for (var _d = true, _e = __asyncValues(this.cfg.hooks.beforeRetry || []), _f; _f = await _e.next(), _a = _f.done, !_a;) {
-                _c = _f.value;
-                _d = false;
-                try {
-                    const hook = _c;
-                    await hook(res);
-                }
-                finally {
-                    _d = true;
-                }
-            }
-        }
-        catch (e_3_1) { e_3 = { error: e_3_1 }; }
-        finally {
-            try {
-                if (!_d && !_a && (_b = _e.return)) await _b.call(_e);
-            }
-            finally { if (e_3) throw e_3.error; }
+        for (const hook of this.cfg.hooks.beforeRetry || []) {
+            await hook(res);
         }
         const { count, timeoutMultiplier, timeoutMax } = res.req.retry;
         if (retryStatus.retryAttempt >= count) {
@@ -334,6 +290,22 @@ export class Fetcher {
         }
         if (retryStatus.retryStopped)
             return;
+        // Here we know that more retries will be attempted
+        // We don't log "last error", because it will be thrown and logged by consumer,
+        // but we should log all previous errors, otherwise they are lost.
+        // Here is the right place where we know it's not the "last error"
+        if (res.err) {
+            const { retryAttempt } = retryStatus;
+            this.cfg.logger.error([
+                ' <<',
+                ((_a = res.fetchResponse) === null || _a === void 0 ? void 0 : _a.status) || 0,
+                res.signature,
+                `try#${retryAttempt + 1}/${count + 1}`,
+                _since(res.req.started),
+            ]
+                .filter(Boolean)
+                .join(' '), res.err.cause || res.err);
+        }
         retryStatus.retryAttempt++;
         retryStatus.retryTimeout = _clamp(retryStatus.retryTimeout * timeoutMultiplier, 0, timeoutMax);
         const noise = Math.random() * 500;
@@ -437,7 +409,7 @@ export class Fetcher {
     normalizeOptions(url, opt) {
         var _a, _b;
         const { timeoutSeconds, throwHttpErrors, retryPost, retry4xx, retry5xx, retry, mode, jsonReviver, } = this.cfg;
-        const req = Object.assign(Object.assign({ mode,
+        const req = Object.assign(Object.assign({ started: Date.now(), mode,
             url,
             timeoutSeconds,
             throwHttpErrors,

package/dist-esm/promise/pMap.js

@@ -6,7 +6,6 @@ Improvements:
 - Included Typescript typings (no need for @types/p-map)
 - Compatible with pProps (that had typings issues)
  */
-import { __asyncValues } from "tslib";
 import { END, ErrorMode, SKIP } from '..';
 /**
  * Returns a `Promise` that is fulfilled when all promises in `input` and ones returned from `mapper` are fulfilled,
@@ -35,7 +34,6 @@ import { END, ErrorMode, SKIP } from '..';
  * })();
  */
 export async function pMap(iterable, mapper, opt = {}) {
-    var _a, e_1, _b, _c;
     const ret = [];
     // const iterator = iterable[Symbol.iterator]()
     const items = [...iterable];
@@ -49,40 +47,23 @@ export async function pMap(iterable, mapper, opt = {}) {
     let currentIndex = 0;
     // Special cases that are able to preserve async stack traces
     if (concurrency === 1) {
-        try {
-            // Special case for concurrency == 1
-            for (var _d = true, items_1 = __asyncValues(items), items_1_1; items_1_1 = await items_1.next(), _a = items_1_1.done, !_a;) {
-                _c = items_1_1.value;
-                _d = false;
-                try {
-                    const item = _c;
-                    try {
-                        const r = await mapper(item, currentIndex++);
-                        if (r === END)
-                            break;
-                        if (r !== SKIP)
-                            ret.push(r);
-                    }
-                    catch (err) {
-                        if (errorMode === ErrorMode.THROW_IMMEDIATELY)
-                            throw err;
-                        if (errorMode === ErrorMode.THROW_AGGREGATED) {
-                            errors.push(err);
-                        }
-                        // otherwise, suppress completely
-                    }
-                }
-                finally {
-                    _d = true;
-                }
-            }
-        }
-        catch (e_1_1) { e_1 = { error: e_1_1 }; }
-        finally {
+        // Special case for concurrency == 1
+        for (const item of items) {
             try {
-                if (!_d && !_a && (_b = items_1.return)) await _b.call(items_1);
+                const r = await mapper(item, currentIndex++);
+                if (r === END)
+                    break;
+                if (r !== SKIP)
+                    ret.push(r);
+            }
+            catch (err) {
+                if (errorMode === ErrorMode.THROW_IMMEDIATELY)
+                    throw err;
+                if (errorMode === ErrorMode.THROW_AGGREGATED) {
+                    errors.push(err);
+                }
+                // otherwise, suppress completely
             }
-            finally { if (e_1) throw e_1.error; }
         }
         if (errors.length) {
             throw new AggregateError(errors, `pMap resulted in ${errors.length} error(s)`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@naturalcycles/js-lib",
-  "version": "14.145.0",
+  "version": "14.146.0",
   "scripts": {
     "prepare": "husky install",
     "build-prod": "build-prod-esm-cjs",

package/src/http/fetcher.model.ts

@@ -1,16 +1,24 @@
 import type { CommonLogger } from '../log/commonLogger'
 import type { Promisable } from '../typeFest'
-import type { Reviver } from '../types'
+import type { Reviver, UnixTimestampMillisNumber } from '../types'
 import type { HttpMethod, HttpStatusFamily } from './http.model'
 
-export interface FetcherNormalizedCfg extends Required<FetcherCfg>, FetcherRequest {
+export interface FetcherNormalizedCfg
+  extends Required<FetcherCfg>,
+    Omit<FetcherRequest, 'started'> {
   logger: CommonLogger
   searchParams: Record<string, any>
 }
 
-export type FetcherBeforeRequestHook = (req: FetcherRequest) => Promisable<void>
-export type FetcherAfterResponseHook = (res: FetcherResponse) => Promisable<void>
-export type FetcherBeforeRetryHook = (res: FetcherResponse) => Promisable<void>
+export type FetcherBeforeRequestHook = <BODY = unknown>(
+  req: FetcherRequest<BODY>,
+) => Promisable<void>
+export type FetcherAfterResponseHook = <BODY = unknown>(
+  res: FetcherResponse<BODY>,
+) => Promisable<void>
+export type FetcherBeforeRetryHook = <BODY = unknown>(
+  res: FetcherResponse<BODY>,
+) => Promisable<void>
 
 export interface FetcherCfg {
   /**
@@ -88,7 +96,8 @@ export interface FetcherRetryOptions {
   timeoutMultiplier: number
 }
 
-export interface FetcherRequest extends Omit<FetcherOptions, 'method' | 'headers' | 'baseUrl'> {
+export interface FetcherRequest<BODY = unknown>
+  extends Omit<FetcherOptions<BODY>, 'method' | 'headers' | 'baseUrl'> {
   url: string
   init: RequestInitNormalized
   mode: FetcherMode
@@ -98,9 +107,10 @@ export interface FetcherRequest extends Omit<FetcherOptions, 'method' | 'headers
   retryPost: boolean
   retry4xx: boolean
   retry5xx: boolean
+  started: UnixTimestampMillisNumber
 }
 
-export interface FetcherOptions {
+export interface FetcherOptions<BODY = unknown> {
   method?: HttpMethod
 
   baseUrl?: string
@@ -159,6 +169,19 @@ export interface FetcherOptions {
   retry5xx?: boolean
 
   jsonReviver?: Reviver
+
+  /**
+   * Allows to walk over multiple pages of results.
+   * Paginate take a function.
+   * Function has access to FetcherRequest and FetcherResponse
+   * and has to make a decision to continue pagination or not.
+   * Return true to continue, false otherwise.
+   * If continue - it is expected to modify/mutate the FetcherRequest, as it will be used
+   * to request the next page.
+   *
+   * @experimental
+   */
+  paginate?: (req: FetcherRequest<BODY>, res: FetcherSuccessResponse<BODY>) => Promisable<boolean>
 }
 
 export type RequestInitNormalized = Omit<RequestInit, 'method' | 'headers'> & {

package/src/http/fetcher.ts

@@ -15,7 +15,6 @@ import {
 import { pDelay } from '../promise/pDelay'
 import { _jsonParse, _jsonParseIfPossible } from '../string/json.util'
 import { _since } from '../time/time.util'
-import { UnixTimestampNumber } from '../types'
 import type {
   FetcherAfterResponseHook,
   FetcherBeforeRequestHook,
@@ -54,7 +53,10 @@ export class Fetcher {
       const m = method.toLowerCase()
 
       // mode=void
-      ;(this as any)[`${m}Void`] = async (url: string, opt?: FetcherOptions): Promise<void> => {
+      ;(this as any)[`${m}Void`] = async (
+        url: string,
+        opt?: FetcherOptions<void>,
+      ): Promise<void> => {
         return await this.fetch<void>(url, {
           method,
           mode: 'void',
@@ -63,7 +65,10 @@ export class Fetcher {
       }
 
       if (method === 'HEAD') return // mode=text
-      ;(this as any)[`${m}Text`] = async (url: string, opt?: FetcherOptions): Promise<string> => {
+      ;(this as any)[`${m}Text`] = async (
+        url: string,
+        opt?: FetcherOptions<string>,
+      ): Promise<string> => {
         return await this.fetch<string>(url, {
           method,
           mode: 'text',
@@ -72,7 +77,7 @@ export class Fetcher {
       }
 
       // Default mode=json, but overridable
-      ;(this as any)[m] = async <T = unknown>(url: string, opt?: FetcherOptions): Promise<T> => {
+      ;(this as any)[m] = async <T = unknown>(url: string, opt?: FetcherOptions<T>): Promise<T> => {
         return await this.fetch<T>(url, {
           method,
           mode: 'json',
@@ -108,26 +113,26 @@ export class Fetcher {
 
   // These methods are generated dynamically in the constructor
   // These default methods use mode=json
-  get!: <T = unknown>(url: string, opt?: FetcherOptions) => Promise<T>
-  post!: <T = unknown>(url: string, opt?: FetcherOptions) => Promise<T>
-  put!: <T = unknown>(url: string, opt?: FetcherOptions) => Promise<T>
-  patch!: <T = unknown>(url: string, opt?: FetcherOptions) => Promise<T>
-  delete!: <T = unknown>(url: string, opt?: FetcherOptions) => Promise<T>
+  get!: <T = unknown>(url: string, opt?: FetcherOptions<T>) => Promise<T>
+  post!: <T = unknown>(url: string, opt?: FetcherOptions<T>) => Promise<T>
+  put!: <T = unknown>(url: string, opt?: FetcherOptions<T>) => Promise<T>
+  patch!: <T = unknown>(url: string, opt?: FetcherOptions<T>) => Promise<T>
+  delete!: <T = unknown>(url: string, opt?: FetcherOptions<T>) => Promise<T>
 
   // mode=text
-  getText!: (url: string, opt?: FetcherOptions) => Promise<string>
-  postText!: (url: string, opt?: FetcherOptions) => Promise<string>
-  putText!: (url: string, opt?: FetcherOptions) => Promise<string>
-  patchText!: (url: string, opt?: FetcherOptions) => Promise<string>
-  deleteText!: (url: string, opt?: FetcherOptions) => Promise<string>
+  getText!: (url: string, opt?: FetcherOptions<string>) => Promise<string>
+  postText!: (url: string, opt?: FetcherOptions<string>) => Promise<string>
+  putText!: (url: string, opt?: FetcherOptions<string>) => Promise<string>
+  patchText!: (url: string, opt?: FetcherOptions<string>) => Promise<string>
+  deleteText!: (url: string, opt?: FetcherOptions<string>) => Promise<string>
 
   // mode=void (no body fetching/parsing)
-  getVoid!: (url: string, opt?: FetcherOptions) => Promise<void>
-  postVoid!: (url: string, opt?: FetcherOptions) => Promise<void>
-  putVoid!: (url: string, opt?: FetcherOptions) => Promise<void>
-  patchVoid!: (url: string, opt?: FetcherOptions) => Promise<void>
-  deleteVoid!: (url: string, opt?: FetcherOptions) => Promise<void>
-  headVoid!: (url: string, opt?: FetcherOptions) => Promise<void>
+  getVoid!: (url: string, opt?: FetcherOptions<void>) => Promise<void>
+  postVoid!: (url: string, opt?: FetcherOptions<void>) => Promise<void>
+  putVoid!: (url: string, opt?: FetcherOptions<void>) => Promise<void>
+  patchVoid!: (url: string, opt?: FetcherOptions<void>) => Promise<void>
+  deleteVoid!: (url: string, opt?: FetcherOptions<void>) => Promise<void>
+  headVoid!: (url: string, opt?: FetcherOptions<void>) => Promise<void>
 
   // mode=readableStream
   /**
@@ -136,14 +141,17 @@ export class Fetcher {
    * More on streams and Node interop:
    * https://css-tricks.com/web-streams-everywhere-and-fetch-for-node-js/
    */
-  async getReadableStream(url: string, opt?: FetcherOptions): Promise<ReadableStream<Uint8Array>> {
+  async getReadableStream(
+    url: string,
+    opt?: FetcherOptions<ReadableStream<Uint8Array>>,
+  ): Promise<ReadableStream<Uint8Array>> {
     return await this.fetch(url, {
       mode: 'readableStream',
       ...opt,
     })
   }
 
-  async fetch<T = unknown>(url: string, opt?: FetcherOptions): Promise<T> {
+  async fetch<T = unknown>(url: string, opt?: FetcherOptions<T>): Promise<T> {
     const res = await this.doFetch<T>(url, opt)
     if (res.err) {
       if (res.req.throwHttpErrors) throw res.err
@@ -159,11 +167,14 @@ export class Fetcher {
    */
   async doFetch<T = unknown>(
     url: string,
-    rawOpt: FetcherOptions = {},
+    opt: FetcherOptions<T> = {},
   ): Promise<FetcherResponse<T>> {
-    const { logger } = this.cfg
+    const req = this.normalizeOptions(url, opt)
+    return await this.doFetchRequest<T>(req)
+  }
 
-    const req = this.normalizeOptions(url, rawOpt)
+  async doFetchRequest<T = unknown>(req: FetcherRequest<T>): Promise<FetcherResponse<T>> {
+    const { logger } = this.cfg
     const {
       timeoutSeconds,
       init: { method },
@@ -179,7 +190,7 @@ export class Fetcher {
       }, timeoutSeconds * 1000) as any as number
     }
 
-    for await (const hook of this.cfg.hooks.beforeRequest || []) {
+    for (const hook of this.cfg.hooks.beforeRequest || []) {
       await hook(req)
     }
 
@@ -199,7 +210,7 @@ export class Fetcher {
     } as FetcherResponse<any>
 
     while (!res.retryStatus.retryStopped) {
-      const started = Date.now()
+      req.started = Date.now()
 
       if (this.cfg.logRequest) {
         const { retryAttempt } = res.retryStatus
@@ -226,27 +237,29 @@ export class Fetcher {
       res.statusFamily = this.getStatusFamily(res)
 
       if (res.fetchResponse?.ok) {
-        await this.onOkResponse(
-          res as FetcherResponse<T> & { fetchResponse: Response },
-          started,
-          timeout,
-        )
+        await this.onOkResponse(res as FetcherResponse<T> & { fetchResponse: Response }, timeout)
       } else {
         // !res.ok
-        await this.onNotOkResponse(res, started, timeout)
+        await this.onNotOkResponse(res, timeout)
       }
     }
 
-    for await (const hook of this.cfg.hooks.afterResponse || []) {
+    for (const hook of this.cfg.hooks.afterResponse || []) {
       await hook(res)
     }
 
+    if (req.paginate && res.ok) {
+      const proceed = await req.paginate(req, res)
+      if (proceed) {
+        return await this.doFetchRequest(req)
+      }
+    }
+
     return res
   }
 
   private async onOkResponse(
     res: FetcherResponse<any> & { fetchResponse: Response },
-    started: UnixTimestampNumber,
     timeout?: number,
   ): Promise<void> {
     const { req } = res
@@ -273,7 +286,7 @@ export class Fetcher {
             res.err = _anyToError(err)
             res.ok = false
 
-            return await this.onNotOkResponse(res, started, timeout)
+            return await this.onNotOkResponse(res, timeout)
           }
         } else {
           // Body had a '' (empty string)
@@ -296,7 +309,7 @@ export class Fetcher {
       if (res.body === null) {
         res.err = new Error(`fetchResponse.body is null`)
         res.ok = false
-        return await this.onNotOkResponse(res, started, timeout)
+        return await this.onNotOkResponse(res, timeout)
       }
     }
 
@@ -313,7 +326,7 @@ export class Fetcher {
           res.fetchResponse.status,
           res.signature,
           retryAttempt && `try#${retryAttempt + 1}/${req.retry.count + 1}`,
-          _since(started),
+          _since(res.req.started),
         ]
           .filter(Boolean)
           .join(' '),
@@ -332,11 +345,7 @@ export class Fetcher {
     return await globalThis.fetch(url, init)
   }
 
-  private async onNotOkResponse(
-    res: FetcherResponse,
-    started: UnixTimestampNumber,
-    timeout?: number,
-  ): Promise<void> {
+  private async onNotOkResponse(res: FetcherResponse, timeout?: number): Promise<void> {
     clearTimeout(timeout)
 
     let cause: ErrorObject | undefined
@@ -367,7 +376,7 @@ export class Fetcher {
         requestBaseUrl: this.cfg.baseUrl || (null as any),
         requestMethod: res.req.init.method,
         requestSignature: res.signature,
-        requestDuration: Date.now() - started,
+        requestDuration: Date.now() - res.req.started,
       }),
       cause,
     )
@@ -382,7 +391,7 @@ export class Fetcher {
       retryStatus.retryStopped = true
     }
 
-    for await (const hook of this.cfg.hooks.beforeRetry || []) {
+    for (const hook of this.cfg.hooks.beforeRetry || []) {
       await hook(res)
     }
 
@@ -394,6 +403,26 @@ export class Fetcher {
 
     if (retryStatus.retryStopped) return
 
+    // Here we know that more retries will be attempted
+    // We don't log "last error", because it will be thrown and logged by consumer,
+    // but we should log all previous errors, otherwise they are lost.
+    // Here is the right place where we know it's not the "last error"
+    if (res.err) {
+      const { retryAttempt } = retryStatus
+      this.cfg.logger.error(
+        [
+          ' <<',
+          res.fetchResponse?.status || 0,
+          res.signature,
+          `try#${retryAttempt + 1}/${count + 1}`,
+          _since(res.req.started),
+        ]
+          .filter(Boolean)
+          .join(' '),
+        res.err.cause || res.err,
+      )
+    }
+
     retryStatus.retryAttempt++
     retryStatus.retryTimeout = _clamp(retryStatus.retryTimeout * timeoutMultiplier, 0, timeoutMax)
 
@@ -499,7 +528,7 @@ export class Fetcher {
     return norm
   }
 
-  private normalizeOptions(url: string, opt: FetcherOptions): FetcherRequest {
+  private normalizeOptions<BODY>(url: string, opt: FetcherOptions<BODY>): FetcherRequest<BODY> {
     const {
       timeoutSeconds,
       throwHttpErrors,
@@ -511,7 +540,8 @@ export class Fetcher {
       jsonReviver,
     } = this.cfg
 
-    const req: FetcherRequest = {
+    const req: FetcherRequest<BODY> = {
+      started: Date.now(),
       mode,
       url,
       timeoutSeconds,

package/src/promise/pMap.ts

@@ -77,7 +77,7 @@ export async function pMap<IN, OUT>(
   if (concurrency === 1) {
     // Special case for concurrency == 1
 
-    for await (const item of items) {
+    for (const item of items) {
       try {
         const r = await mapper(item, currentIndex++)
         if (r === END) break