@naturalcycles/js-lib 14.146.0 → 14.148.0

package/dist/http/fetcher.d.ts

@@ -1,5 +1,5 @@
 /// <reference lib="dom" />
-import type { FetcherAfterResponseHook, FetcherBeforeRequestHook, FetcherBeforeRetryHook, FetcherCfg, FetcherNormalizedCfg, FetcherOptions, FetcherRequest, FetcherResponse } from './fetcher.model';
+import type { FetcherAfterResponseHook, FetcherBeforeRequestHook, FetcherBeforeRetryHook, FetcherCfg, FetcherNormalizedCfg, FetcherOptions, FetcherResponse } from './fetcher.model';
 /**
  * Experimental wrapper around Fetch.
  * Works in both Browser and Node, using `globalThis.fetch`.
@@ -37,14 +37,13 @@ export declare class Fetcher {
      * https://css-tricks.com/web-streams-everywhere-and-fetch-for-node-js/
      */
     getReadableStream(url: string, opt?: FetcherOptions<ReadableStream<Uint8Array>>): Promise<ReadableStream<Uint8Array>>;
-    fetch<T = unknown>(url: string, opt?: FetcherOptions<T>): Promise<T>;
+    fetch<T = unknown>(opt: FetcherOptions<T>): Promise<T>;
     /**
      * Returns FetcherResponse.
      * Never throws, returns `err` property in the response instead.
      * Use this method instead of `throwHttpErrors: false` or try-catching.
      */
-    doFetch<T = unknown>(url: string, opt?: FetcherOptions<T>): Promise<FetcherResponse<T>>;
-    doFetchRequest<T = unknown>(req: FetcherRequest<T>): Promise<FetcherResponse<T>>;
+    doFetch<T = unknown>(opt: FetcherOptions<T>): Promise<FetcherResponse<T>>;
     private onOkResponse;
     /**
      * This method exists to be able to easily mock it.
@@ -52,6 +51,7 @@ export declare class Fetcher {
     callNativeFetch(url: string, init: RequestInit): Promise<Response>;
     private onNotOkResponse;
     private processRetry;
+    private getRetryTimeout;
     /**
      * Default is yes,
      * unless there's reason not to (e.g method is POST).

package/dist/http/fetcher.js

@@ -31,7 +31,8 @@ class Fetcher {
         http_model_1.HTTP_METHODS.forEach(method => {
             const m = method.toLowerCase();
             this[`${m}Void`] = async (url, opt) => {
-                return await this.fetch(url, {
+                return await this.fetch({
+                    url,
                     method,
                     mode: 'void',
                     ...opt,
@@ -41,14 +42,16 @@ class Fetcher {
                 return // mode=text
                 ;
             this[`${m}Text`] = async (url, opt) => {
-                return await this.fetch(url, {
+                return await this.fetch({
+                    url,
                     method,
                     mode: 'text',
                     ...opt,
                 });
             };
             this[m] = async (url, opt) => {
-                return await this.fetch(url, {
+                return await this.fetch({
+                    url,
                     method,
                     mode: 'json',
                     ...opt,
@@ -85,13 +88,14 @@ class Fetcher {
      * https://css-tricks.com/web-streams-everywhere-and-fetch-for-node-js/
      */
     async getReadableStream(url, opt) {
-        return await this.fetch(url, {
+        return await this.fetch({
+            url,
             mode: 'readableStream',
             ...opt,
         });
     }
-    async fetch(url, opt) {
-        const res = await this.doFetch(url, opt);
+    async fetch(opt) {
+        const res = await this.doFetch(opt);
         if (res.err) {
             if (res.req.throwHttpErrors)
                 throw res.err;
@@ -104,11 +108,8 @@ class Fetcher {
      * Never throws, returns `err` property in the response instead.
      * Use this method instead of `throwHttpErrors: false` or try-catching.
      */
-    async doFetch(url, opt = {}) {
-        const req = this.normalizeOptions(url, opt);
-        return await this.doFetchRequest(req);
-    }
-    async doFetchRequest(req) {
+    async doFetch(opt) {
+        const req = this.normalizeOptions(opt);
         const { logger } = this.cfg;
         const { timeoutSeconds, init: { method }, } = req;
         // setup timeout
@@ -123,9 +124,9 @@ class Fetcher {
         for (const hook of this.cfg.hooks.beforeRequest || []) {
             await hook(req);
         }
-        const isFullUrl = req.url.includes('://');
-        const fullUrl = isFullUrl ? new URL(req.url) : undefined;
-        const shortUrl = fullUrl ? this.getShortUrl(fullUrl) : req.url;
+        const isFullUrl = req.fullUrl.includes('://');
+        const fullUrl = isFullUrl ? new URL(req.fullUrl) : undefined;
+        const shortUrl = fullUrl ? this.getShortUrl(fullUrl) : req.fullUrl;
         const signature = [method, shortUrl].join(' ');
         const res = {
             req,
@@ -148,8 +149,10 @@ class Fetcher {
                 }
             }
             try {
-                res.fetchResponse = await this.callNativeFetch(req.url, req.init);
+                res.fetchResponse = await this.callNativeFetch(req.fullUrl, req.init);
                 res.ok = res.fetchResponse.ok;
+                // important to set it to undefined, otherwise it can keep the previous value (from previous try)
+                res.err = undefined;
             }
             catch (err) {
                 // For example, CORS error would result in "TypeError: failed to fetch" here
@@ -171,9 +174,9 @@ class Fetcher {
             await hook(res);
         }
         if (req.paginate && res.ok) {
-            const proceed = await req.paginate(req, res);
-            if (proceed) {
-                return await this.doFetchRequest(req);
+            const proceeed = await req.paginate(res, opt);
+            if (proceeed) {
+                return await this.doFetch(opt);
             }
         }
         return res;
@@ -280,7 +283,7 @@ class Fetcher {
             // Enabled, cause `data` is not printed by default when error is HttpError
             // method: req.method,
             // tryCount: req.tryCount,
-            requestUrl: res.req.url,
+            requestUrl: res.req.fullUrl,
             requestBaseUrl: this.cfg.baseUrl || null,
             requestMethod: res.req.init.method,
             requestSignature: res.signature,
@@ -307,12 +310,11 @@ class Fetcher {
         // but we should log all previous errors, otherwise they are lost.
         // Here is the right place where we know it's not the "last error"
         if (res.err) {
-            const { retryAttempt } = retryStatus;
             this.cfg.logger.error([
                 ' <<',
                 res.fetchResponse?.status || 0,
                 res.signature,
-                `try#${retryAttempt + 1}/${count + 1}`,
+                `try#${retryStatus.retryAttempt + 1}/${count + 1}`,
                 (0, time_util_1._since)(res.req.started),
             ]
                 .filter(Boolean)
@@ -320,8 +322,36 @@ class Fetcher {
         }
         retryStatus.retryAttempt++;
         retryStatus.retryTimeout = (0, number_util_1._clamp)(retryStatus.retryTimeout * timeoutMultiplier, 0, timeoutMax);
-        const noise = Math.random() * 500;
-        await (0, pDelay_1.pDelay)(retryStatus.retryTimeout + noise);
+        await (0, pDelay_1.pDelay)(this.getRetryTimeout(res));
+    }
+    getRetryTimeout(res) {
+        let timeout = 0;
+        // Handling http 429 with specific retry headers
+        // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
+        if (res.fetchResponse && [429, 503].includes(res.fetchResponse.status)) {
+            const retryAfterStr = res.fetchResponse.headers.get('retry-after') ??
+                res.fetchResponse.headers.get('x-ratelimit-reset');
+            if (retryAfterStr) {
+                if (Number(retryAfterStr)) {
+                    timeout = Number(retryAfterStr) * 1000;
+                }
+                else {
+                    const date = new Date(retryAfterStr);
+                    if (!isNaN(date)) {
+                        timeout = Number(date) - Date.now();
+                    }
+                }
+                this.cfg.logger.log(`retry-after: ${retryAfterStr}`);
+                if (!timeout) {
+                    this.cfg.logger.warn(`retry-after could not be parsed`);
+                }
+            }
+        }
+        if (!timeout) {
+            const noise = Math.random() * 500;
+            timeout = res.retryStatus.retryTimeout + noise;
+        }
+        return timeout;
     }
     /**
      * Default is yes,
@@ -387,7 +417,7 @@ class Fetcher {
         const { debug = false } = cfg;
         const norm = (0, object_util_1._merge)({
             baseUrl: '',
-            url: '',
+            inputUrl: '',
             mode: 'void',
             searchParams: {},
             timeoutSeconds: 30,
@@ -415,12 +445,11 @@ class Fetcher {
         norm.init.headers = (0, object_util_1._mapKeys)(norm.init.headers, k => k.toLowerCase());
         return norm;
     }
-    normalizeOptions(url, opt) {
+    normalizeOptions(opt) {
         const { timeoutSeconds, throwHttpErrors, retryPost, retry4xx, retry5xx, retry, mode, jsonReviver, } = this.cfg;
         const req = {
             started: Date.now(),
             mode,
-            url,
             timeoutSeconds,
             throwHttpErrors,
             retryPost,
@@ -428,6 +457,8 @@ class Fetcher {
             retry5xx,
             jsonReviver,
             ...(0, object_util_1._omit)(opt, ['method', 'headers', 'credentials']),
+            inputUrl: opt.url || '',
+            fullUrl: opt.url || '',
             retry: {
                 ...retry,
                 ...(0, object_util_1._filterUndefinedValues)(opt.retry || {}),
@@ -444,11 +475,11 @@ class Fetcher {
         // setup url
         const baseUrl = opt.baseUrl || this.cfg.baseUrl;
         if (baseUrl) {
-            if (url.startsWith('/')) {
+            if (req.fullUrl.startsWith('/')) {
                 console.warn(`Fetcher: url should not start with / when baseUrl is specified`);
-                url = url.slice(1);
+                req.fullUrl = req.fullUrl.slice(1);
             }
-            req.url = `${baseUrl}/${url}`;
+            req.fullUrl = `${baseUrl}/${req.inputUrl}`;
         }
         const searchParams = (0, object_util_1._filterUndefinedValues)({
             ...this.cfg.searchParams,
@@ -456,7 +487,7 @@ class Fetcher {
         });
         if (Object.keys(searchParams).length) {
             const qs = new URLSearchParams(searchParams).toString();
-            req.url += req.url.includes('?') ? '&' : '?' + qs;
+            req.fullUrl += req.fullUrl.includes('?') ? '&' : '?' + qs;
         }
         // setup request body
         if (opt.json !== undefined) {

package/dist/http/fetcher.model.d.ts

@@ -2,7 +2,7 @@ import type { CommonLogger } from '../log/commonLogger';
 import type { Promisable } from '../typeFest';
 import type { Reviver, UnixTimestampMillisNumber } from '../types';
 import type { HttpMethod, HttpStatusFamily } from './http.model';
-export interface FetcherNormalizedCfg extends Required<FetcherCfg>, Omit<FetcherRequest, 'started'> {
+export interface FetcherNormalizedCfg extends Required<FetcherCfg>, Omit<FetcherRequest, 'started' | 'fullUrl'> {
     logger: CommonLogger;
     searchParams: Record<string, any>;
 }
@@ -77,8 +77,16 @@ export interface FetcherRetryOptions {
     timeoutMax: number;
     timeoutMultiplier: number;
 }
-export interface FetcherRequest<BODY = unknown> extends Omit<FetcherOptions<BODY>, 'method' | 'headers' | 'baseUrl'> {
-    url: string;
+export interface FetcherRequest<BODY = unknown> extends Omit<FetcherOptions<BODY>, 'method' | 'headers' | 'baseUrl' | 'url'> {
+    /**
+     * inputUrl is only the part that was passed in the request,
+     * without baseUrl or searchParams.
+     */
+    inputUrl: string;
+    /**
+     * fullUrl includes baseUrl and searchParams.
+     */
+    fullUrl: string;
     init: RequestInitNormalized;
     mode: FetcherMode;
     throwHttpErrors: boolean;
@@ -91,6 +99,11 @@ export interface FetcherRequest<BODY = unknown> extends Omit<FetcherOptions<BODY
 }
 export interface FetcherOptions<BODY = unknown> {
     method?: HttpMethod;
+    /**
+     * If defined - this `url` will override the original given `url`.
+     * baseUrl (and searchParams) will still modify it.
+     */
+    url?: string;
     baseUrl?: string;
     throwHttpErrors?: boolean;
     /**
@@ -138,15 +151,18 @@ export interface FetcherOptions<BODY = unknown> {
     /**
      * Allows to walk over multiple pages of results.
      * Paginate take a function.
-     * Function has access to FetcherRequest and FetcherResponse
+     * Function has access to FetcherResponse and FetcherOptions
      * and has to make a decision to continue pagination or not.
-     * Return true to continue, false otherwise.
-     * If continue - it is expected to modify/mutate the FetcherRequest, as it will be used
-     * to request the next page.
+     *
+     * Return false to stop pagination.
+     * Return true to continue pagination.
+     * Feel free to mutate/modify opt (FetcherOptions), for example:
+     *
+     * opt.searchParams!['page']++
      *
      * @experimental
      */
-    paginate?: (req: FetcherRequest<BODY>, res: FetcherSuccessResponse<BODY>) => Promisable<boolean>;
+    paginate?: (res: FetcherSuccessResponse<BODY>, opt: FetcherOptions<BODY>) => Promisable<boolean>;
 }
 export type RequestInitNormalized = Omit<RequestInit, 'method' | 'headers'> & {
     method: HttpMethod;

package/dist-esm/http/fetcher.js

@@ -28,16 +28,19 @@ export class Fetcher {
         HTTP_METHODS.forEach(method => {
             const m = method.toLowerCase();
             this[`${m}Void`] = async (url, opt) => {
-                return await this.fetch(url, Object.assign({ method, mode: 'void' }, opt));
+                return await this.fetch(Object.assign({ url,
+                    method, mode: 'void' }, opt));
             };
             if (method === 'HEAD')
                 return // mode=text
                 ;
             this[`${m}Text`] = async (url, opt) => {
-                return await this.fetch(url, Object.assign({ method, mode: 'text' }, opt));
+                return await this.fetch(Object.assign({ url,
+                    method, mode: 'text' }, opt));
             };
             this[m] = async (url, opt) => {
-                return await this.fetch(url, Object.assign({ method, mode: 'json' }, opt));
+                return await this.fetch(Object.assign({ url,
+                    method, mode: 'json' }, opt));
             };
         });
     }
@@ -73,10 +76,10 @@ export class Fetcher {
      * https://css-tricks.com/web-streams-everywhere-and-fetch-for-node-js/
      */
     async getReadableStream(url, opt) {
-        return await this.fetch(url, Object.assign({ mode: 'readableStream' }, opt));
+        return await this.fetch(Object.assign({ url, mode: 'readableStream' }, opt));
     }
-    async fetch(url, opt) {
-        const res = await this.doFetch(url, opt);
+    async fetch(opt) {
+        const res = await this.doFetch(opt);
         if (res.err) {
             if (res.req.throwHttpErrors)
                 throw res.err;
@@ -89,12 +92,9 @@ export class Fetcher {
      * Never throws, returns `err` property in the response instead.
      * Use this method instead of `throwHttpErrors: false` or try-catching.
      */
-    async doFetch(url, opt = {}) {
-        const req = this.normalizeOptions(url, opt);
-        return await this.doFetchRequest(req);
-    }
-    async doFetchRequest(req) {
+    async doFetch(opt) {
         var _a;
+        const req = this.normalizeOptions(opt);
         const { logger } = this.cfg;
         const { timeoutSeconds, init: { method }, } = req;
         // setup timeout
@@ -109,9 +109,9 @@ export class Fetcher {
         for (const hook of this.cfg.hooks.beforeRequest || []) {
             await hook(req);
         }
-        const isFullUrl = req.url.includes('://');
-        const fullUrl = isFullUrl ? new URL(req.url) : undefined;
-        const shortUrl = fullUrl ? this.getShortUrl(fullUrl) : req.url;
+        const isFullUrl = req.fullUrl.includes('://');
+        const fullUrl = isFullUrl ? new URL(req.fullUrl) : undefined;
+        const shortUrl = fullUrl ? this.getShortUrl(fullUrl) : req.fullUrl;
         const signature = [method, shortUrl].join(' ');
         const res = {
             req,
@@ -134,8 +134,10 @@ export class Fetcher {
                 }
             }
             try {
-                res.fetchResponse = await this.callNativeFetch(req.url, req.init);
+                res.fetchResponse = await this.callNativeFetch(req.fullUrl, req.init);
                 res.ok = res.fetchResponse.ok;
+                // important to set it to undefined, otherwise it can keep the previous value (from previous try)
+                res.err = undefined;
             }
             catch (err) {
                 // For example, CORS error would result in "TypeError: failed to fetch" here
@@ -157,9 +159,9 @@ export class Fetcher {
             await hook(res);
         }
         if (req.paginate && res.ok) {
-            const proceed = await req.paginate(req, res);
-            if (proceed) {
-                return await this.doFetchRequest(req);
+            const proceeed = await req.paginate(res, opt);
+            if (proceeed) {
+                return await this.doFetch(opt);
             }
         }
         return res;
@@ -267,7 +269,7 @@ export class Fetcher {
             // Enabled, cause `data` is not printed by default when error is HttpError
             // method: req.method,
             // tryCount: req.tryCount,
-            requestUrl: res.req.url,
+            requestUrl: res.req.fullUrl,
             requestBaseUrl: this.cfg.baseUrl || null,
             requestMethod: res.req.init.method,
             requestSignature: res.signature,
@@ -295,12 +297,11 @@ export class Fetcher {
         // but we should log all previous errors, otherwise they are lost.
         // Here is the right place where we know it's not the "last error"
         if (res.err) {
-            const { retryAttempt } = retryStatus;
             this.cfg.logger.error([
                 ' <<',
                 ((_a = res.fetchResponse) === null || _a === void 0 ? void 0 : _a.status) || 0,
                 res.signature,
-                `try#${retryAttempt + 1}/${count + 1}`,
+                `try#${retryStatus.retryAttempt + 1}/${count + 1}`,
                 _since(res.req.started),
             ]
                 .filter(Boolean)
@@ -308,8 +309,36 @@ export class Fetcher {
         }
         retryStatus.retryAttempt++;
         retryStatus.retryTimeout = _clamp(retryStatus.retryTimeout * timeoutMultiplier, 0, timeoutMax);
-        const noise = Math.random() * 500;
-        await pDelay(retryStatus.retryTimeout + noise);
+        await pDelay(this.getRetryTimeout(res));
+    }
+    getRetryTimeout(res) {
+        var _a;
+        let timeout = 0;
+        // Handling http 429 with specific retry headers
+        // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
+        if (res.fetchResponse && [429, 503].includes(res.fetchResponse.status)) {
+            const retryAfterStr = (_a = res.fetchResponse.headers.get('retry-after')) !== null && _a !== void 0 ? _a : res.fetchResponse.headers.get('x-ratelimit-reset');
+            if (retryAfterStr) {
+                if (Number(retryAfterStr)) {
+                    timeout = Number(retryAfterStr) * 1000;
+                }
+                else {
+                    const date = new Date(retryAfterStr);
+                    if (!isNaN(date)) {
+                        timeout = Number(date) - Date.now();
+                    }
+                }
+                this.cfg.logger.log(`retry-after: ${retryAfterStr}`);
+                if (!timeout) {
+                    this.cfg.logger.warn(`retry-after could not be parsed`);
+                }
+            }
+        }
+        if (!timeout) {
+            const noise = Math.random() * 500;
+            timeout = res.retryStatus.retryTimeout + noise;
+        }
+        return timeout;
     }
     /**
      * Default is yes,
@@ -378,7 +407,7 @@ export class Fetcher {
         const { debug = false } = cfg;
         const norm = _merge({
             baseUrl: '',
-            url: '',
+            inputUrl: '',
             mode: 'void',
             searchParams: {},
             timeoutSeconds: 30,
@@ -406,32 +435,31 @@ export class Fetcher {
         norm.init.headers = _mapKeys(norm.init.headers, k => k.toLowerCase());
         return norm;
     }
-    normalizeOptions(url, opt) {
+    normalizeOptions(opt) {
         var _a, _b;
         const { timeoutSeconds, throwHttpErrors, retryPost, retry4xx, retry5xx, retry, mode, jsonReviver, } = this.cfg;
         const req = Object.assign(Object.assign({ started: Date.now(), mode,
-            url,
             timeoutSeconds,
             throwHttpErrors,
             retryPost,
             retry4xx,
             retry5xx,
-            jsonReviver }, _omit(opt, ['method', 'headers', 'credentials'])), { retry: Object.assign(Object.assign({}, retry), _filterUndefinedValues(opt.retry || {})), init: _merge(Object.assign(Object.assign({}, this.cfg.init), { method: opt.method || this.cfg.init.method, credentials: opt.credentials || this.cfg.init.credentials, redirect: ((_b = (_a = opt.followRedirects) !== null && _a !== void 0 ? _a : this.cfg.followRedirects) !== null && _b !== void 0 ? _b : true) ? 'follow' : 'error' }), {
+            jsonReviver }, _omit(opt, ['method', 'headers', 'credentials'])), { inputUrl: opt.url || '', fullUrl: opt.url || '', retry: Object.assign(Object.assign({}, retry), _filterUndefinedValues(opt.retry || {})), init: _merge(Object.assign(Object.assign({}, this.cfg.init), { method: opt.method || this.cfg.init.method, credentials: opt.credentials || this.cfg.init.credentials, redirect: ((_b = (_a = opt.followRedirects) !== null && _a !== void 0 ? _a : this.cfg.followRedirects) !== null && _b !== void 0 ? _b : true) ? 'follow' : 'error' }), {
                 headers: _mapKeys(opt.headers || {}, k => k.toLowerCase()),
             }) });
         // setup url
         const baseUrl = opt.baseUrl || this.cfg.baseUrl;
         if (baseUrl) {
-            if (url.startsWith('/')) {
+            if (req.fullUrl.startsWith('/')) {
                 console.warn(`Fetcher: url should not start with / when baseUrl is specified`);
-                url = url.slice(1);
+                req.fullUrl = req.fullUrl.slice(1);
             }
-            req.url = `${baseUrl}/${url}`;
+            req.fullUrl = `${baseUrl}/${req.inputUrl}`;
         }
         const searchParams = _filterUndefinedValues(Object.assign(Object.assign({}, this.cfg.searchParams), opt.searchParams));
         if (Object.keys(searchParams).length) {
             const qs = new URLSearchParams(searchParams).toString();
-            req.url += req.url.includes('?') ? '&' : '?' + qs;
+            req.fullUrl += req.fullUrl.includes('?') ? '&' : '?' + qs;
         }
         // setup request body
         if (opt.json !== undefined) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@naturalcycles/js-lib",
-  "version": "14.146.0",
+  "version": "14.148.0",
   "scripts": {
     "prepare": "husky install",
     "build-prod": "build-prod-esm-cjs",

package/src/http/fetcher.model.ts

@@ -5,7 +5,7 @@ import type { HttpMethod, HttpStatusFamily } from './http.model'
 
 export interface FetcherNormalizedCfg
   extends Required<FetcherCfg>,
-    Omit<FetcherRequest, 'started'> {
+    Omit<FetcherRequest, 'started' | 'fullUrl'> {
   logger: CommonLogger
   searchParams: Record<string, any>
 }
@@ -97,8 +97,16 @@ export interface FetcherRetryOptions {
 }
 
 export interface FetcherRequest<BODY = unknown>
-  extends Omit<FetcherOptions<BODY>, 'method' | 'headers' | 'baseUrl'> {
-  url: string
+  extends Omit<FetcherOptions<BODY>, 'method' | 'headers' | 'baseUrl' | 'url'> {
+  /**
+   * inputUrl is only the part that was passed in the request,
+   * without baseUrl or searchParams.
+   */
+  inputUrl: string
+  /**
+   * fullUrl includes baseUrl and searchParams.
+   */
+  fullUrl: string
   init: RequestInitNormalized
   mode: FetcherMode
   throwHttpErrors: boolean
@@ -113,6 +121,12 @@ export interface FetcherRequest<BODY = unknown>
 export interface FetcherOptions<BODY = unknown> {
   method?: HttpMethod
 
+  /**
+   * If defined - this `url` will override the original given `url`.
+   * baseUrl (and searchParams) will still modify it.
+   */
+  url?: string
+
   baseUrl?: string
 
   throwHttpErrors?: boolean
@@ -173,15 +187,18 @@ export interface FetcherOptions<BODY = unknown> {
   /**
    * Allows to walk over multiple pages of results.
    * Paginate take a function.
-   * Function has access to FetcherRequest and FetcherResponse
+   * Function has access to FetcherResponse and FetcherOptions
    * and has to make a decision to continue pagination or not.
-   * Return true to continue, false otherwise.
-   * If continue - it is expected to modify/mutate the FetcherRequest, as it will be used
-   * to request the next page.
+   *
+   * Return false to stop pagination.
+   * Return true to continue pagination.
+   * Feel free to mutate/modify opt (FetcherOptions), for example:
+   *
+   * opt.searchParams!['page']++
    *
    * @experimental
    */
-  paginate?: (req: FetcherRequest<BODY>, res: FetcherSuccessResponse<BODY>) => Promisable<boolean>
+  paginate?: (res: FetcherSuccessResponse<BODY>, opt: FetcherOptions<BODY>) => Promisable<boolean>
 }
 
 export type RequestInitNormalized = Omit<RequestInit, 'method' | 'headers'> & {

package/src/http/fetcher.ts

@@ -15,6 +15,7 @@ import {
 import { pDelay } from '../promise/pDelay'
 import { _jsonParse, _jsonParseIfPossible } from '../string/json.util'
 import { _since } from '../time/time.util'
+import { NumberOfMilliseconds } from '../types'
 import type {
   FetcherAfterResponseHook,
   FetcherBeforeRequestHook,
@@ -57,7 +58,8 @@ export class Fetcher {
         url: string,
         opt?: FetcherOptions<void>,
       ): Promise<void> => {
-        return await this.fetch<void>(url, {
+        return await this.fetch<void>({
+          url,
           method,
           mode: 'void',
           ...opt,
@@ -69,7 +71,8 @@ export class Fetcher {
         url: string,
         opt?: FetcherOptions<string>,
       ): Promise<string> => {
-        return await this.fetch<string>(url, {
+        return await this.fetch<string>({
+          url,
           method,
           mode: 'text',
           ...opt,
@@ -78,7 +81,8 @@ export class Fetcher {
 
       // Default mode=json, but overridable
       ;(this as any)[m] = async <T = unknown>(url: string, opt?: FetcherOptions<T>): Promise<T> => {
-        return await this.fetch<T>(url, {
+        return await this.fetch<T>({
+          url,
           method,
           mode: 'json',
           ...opt,
@@ -145,14 +149,15 @@ export class Fetcher {
     url: string,
     opt?: FetcherOptions<ReadableStream<Uint8Array>>,
   ): Promise<ReadableStream<Uint8Array>> {
-    return await this.fetch(url, {
+    return await this.fetch({
+      url,
       mode: 'readableStream',
       ...opt,
     })
   }
 
-  async fetch<T = unknown>(url: string, opt?: FetcherOptions<T>): Promise<T> {
-    const res = await this.doFetch<T>(url, opt)
+  async fetch<T = unknown>(opt: FetcherOptions<T>): Promise<T> {
+    const res = await this.doFetch<T>(opt)
     if (res.err) {
       if (res.req.throwHttpErrors) throw res.err
       return res as any
@@ -165,15 +170,8 @@ export class Fetcher {
    * Never throws, returns `err` property in the response instead.
    * Use this method instead of `throwHttpErrors: false` or try-catching.
    */
-  async doFetch<T = unknown>(
-    url: string,
-    opt: FetcherOptions<T> = {},
-  ): Promise<FetcherResponse<T>> {
-    const req = this.normalizeOptions(url, opt)
-    return await this.doFetchRequest<T>(req)
-  }
-
-  async doFetchRequest<T = unknown>(req: FetcherRequest<T>): Promise<FetcherResponse<T>> {
+  async doFetch<T = unknown>(opt: FetcherOptions<T>): Promise<FetcherResponse<T>> {
+    const req = this.normalizeOptions(opt)
     const { logger } = this.cfg
     const {
       timeoutSeconds,
@@ -194,9 +192,9 @@ export class Fetcher {
       await hook(req)
     }
 
-    const isFullUrl = req.url.includes('://')
-    const fullUrl = isFullUrl ? new URL(req.url) : undefined
-    const shortUrl = fullUrl ? this.getShortUrl(fullUrl) : req.url
+    const isFullUrl = req.fullUrl.includes('://')
+    const fullUrl = isFullUrl ? new URL(req.fullUrl) : undefined
+    const shortUrl = fullUrl ? this.getShortUrl(fullUrl) : req.fullUrl
     const signature = [method, shortUrl].join(' ')
 
     const res = {
@@ -225,8 +223,10 @@ export class Fetcher {
       }
 
       try {
-        res.fetchResponse = await this.callNativeFetch(req.url, req.init)
+        res.fetchResponse = await this.callNativeFetch(req.fullUrl, req.init)
         res.ok = res.fetchResponse.ok
+        // important to set it to undefined, otherwise it can keep the previous value (from previous try)
+        res.err = undefined
       } catch (err) {
         // For example, CORS error would result in "TypeError: failed to fetch" here
         res.err = err as Error
@@ -249,9 +249,9 @@ export class Fetcher {
     }
 
     if (req.paginate && res.ok) {
-      const proceed = await req.paginate(req, res)
-      if (proceed) {
-        return await this.doFetchRequest(req)
+      const proceeed = await req.paginate(res, opt)
+      if (proceeed) {
+        return await this.doFetch(opt)
       }
     }
 
@@ -372,7 +372,7 @@ export class Fetcher {
         // Enabled, cause `data` is not printed by default when error is HttpError
         // method: req.method,
         // tryCount: req.tryCount,
-        requestUrl: res.req.url,
+        requestUrl: res.req.fullUrl,
         requestBaseUrl: this.cfg.baseUrl || (null as any),
         requestMethod: res.req.init.method,
         requestSignature: res.signature,
@@ -408,13 +408,12 @@ export class Fetcher {
     // but we should log all previous errors, otherwise they are lost.
     // Here is the right place where we know it's not the "last error"
     if (res.err) {
-      const { retryAttempt } = retryStatus
       this.cfg.logger.error(
         [
           ' <<',
           res.fetchResponse?.status || 0,
           res.signature,
-          `try#${retryAttempt + 1}/${count + 1}`,
+          `try#${retryStatus.retryAttempt + 1}/${count + 1}`,
           _since(res.req.started),
         ]
           .filter(Boolean)
@@ -426,8 +425,41 @@ export class Fetcher {
     retryStatus.retryAttempt++
     retryStatus.retryTimeout = _clamp(retryStatus.retryTimeout * timeoutMultiplier, 0, timeoutMax)
 
-    const noise = Math.random() * 500
-    await pDelay(retryStatus.retryTimeout + noise)
+    await pDelay(this.getRetryTimeout(res))
+  }
+
+  private getRetryTimeout(res: FetcherResponse): NumberOfMilliseconds {
+    let timeout: NumberOfMilliseconds = 0
+
+    // Handling http 429 with specific retry headers
+    // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
+    if (res.fetchResponse && [429, 503].includes(res.fetchResponse.status)) {
+      const retryAfterStr =
+        res.fetchResponse.headers.get('retry-after') ??
+        res.fetchResponse.headers.get('x-ratelimit-reset')
+      if (retryAfterStr) {
+        if (Number(retryAfterStr)) {
+          timeout = Number(retryAfterStr) * 1000
+        } else {
+          const date = new Date(retryAfterStr)
+          if (!isNaN(date as any)) {
+            timeout = Number(date) - Date.now()
+          }
+        }
+
+        this.cfg.logger.log(`retry-after: ${retryAfterStr}`)
+        if (!timeout) {
+          this.cfg.logger.warn(`retry-after could not be parsed`)
+        }
+      }
+    }
+
+    if (!timeout) {
+      const noise = Math.random() * 500
+      timeout = res.retryStatus.retryTimeout + noise
+    }
+
+    return timeout
   }
 
   /**
@@ -495,7 +527,7 @@ export class Fetcher {
     const norm: FetcherNormalizedCfg = _merge(
       {
         baseUrl: '',
-        url: '',
+        inputUrl: '',
         mode: 'void',
         searchParams: {},
         timeoutSeconds: 30,
@@ -528,7 +560,7 @@ export class Fetcher {
     return norm
   }
 
-  private normalizeOptions<BODY>(url: string, opt: FetcherOptions<BODY>): FetcherRequest<BODY> {
+  private normalizeOptions<BODY>(opt: FetcherOptions<BODY>): FetcherRequest<BODY> {
     const {
       timeoutSeconds,
       throwHttpErrors,
@@ -543,7 +575,6 @@ export class Fetcher {
     const req: FetcherRequest<BODY> = {
       started: Date.now(),
       mode,
-      url,
       timeoutSeconds,
       throwHttpErrors,
       retryPost,
@@ -551,6 +582,8 @@ export class Fetcher {
       retry5xx,
       jsonReviver,
       ..._omit(opt, ['method', 'headers', 'credentials']),
+      inputUrl: opt.url || '',
+      fullUrl: opt.url || '',
       retry: {
         ...retry,
         ..._filterUndefinedValues(opt.retry || {}),
@@ -571,11 +604,11 @@ export class Fetcher {
     // setup url
     const baseUrl = opt.baseUrl || this.cfg.baseUrl
     if (baseUrl) {
-      if (url.startsWith('/')) {
+      if (req.fullUrl.startsWith('/')) {
         console.warn(`Fetcher: url should not start with / when baseUrl is specified`)
-        url = url.slice(1)
+        req.fullUrl = req.fullUrl.slice(1)
       }
-      req.url = `${baseUrl}/${url}`
+      req.fullUrl = `${baseUrl}/${req.inputUrl}`
     }
 
     const searchParams = _filterUndefinedValues({
@@ -585,7 +618,7 @@ export class Fetcher {
 
     if (Object.keys(searchParams).length) {
       const qs = new URLSearchParams(searchParams).toString()
-      req.url += req.url.includes('?') ? '&' : '?' + qs
+      req.fullUrl += req.fullUrl.includes('?') ? '&' : '?' + qs
     }
 
     // setup request body