@rabbitio/ui-kit 1.0.0-beta.9 → 1.0.0-beta.91

package/src/robust-api-caller/concurrentCalculationsMetadataHolder.js

@@ -0,0 +1,75 @@
+import { v4 } from "uuid";
+
+import { Logger } from "../common-apis/utils/logging/logger.js";
+
+// TODO: [refactoring, low] Consider removing this logic task_id=c360f2af75764bde8badd9ff1cc00d48
+export class ConcurrentCalculationsMetadataHolder {
+    constructor() {
+        this._calculations = {};
+    }
+
+    startCalculation(domain, calculationsHistoryMaxLength = 100) {
+        if (!this._calculations[domain]) {
+            this._calculations[domain] = [];
+        }
+
+        if (this._calculations[domain].length > calculationsHistoryMaxLength) {
+            this._calculations[domain] = this._calculations[domain].slice(
+                Math.round(calculationsHistoryMaxLength * 0.2)
+            );
+        }
+
+        const newCalculation = {
+            startTimestamp: Date.now(),
+            endTimestamp: null,
+            uuid: v4(),
+        };
+
+        this._calculations[domain].push(newCalculation);
+
+        return newCalculation.uuid;
+    }
+
+    endCalculation(domain, uuid, isFailed = false) {
+        try {
+            const calculation = this._calculations[domain].find(calculation => calculation?.uuid === uuid);
+            if (calculation) {
+                calculation.endTimestamp = Date.now();
+                calculation.isFiled = isFailed;
+            }
+
+            const elapsed = (((calculation?.endTimestamp ?? 0) - (calculation?.startTimestamp ?? 0)) / 1000).toFixed(1);
+            Logger.log("endCalculation", `${elapsed} ms: ${domain}.${(calculation?.uuid ?? "").slice(0, 7)}`);
+
+            return calculation;
+        } catch (e) {
+            Logger.logError(e, "endCalculation");
+        }
+    }
+
+    isCalculationLate(domain, uuid) {
+        const queue = this._calculations[domain];
+        const analysingCalculation = queue.find(item => item.uuid === uuid);
+        return (
+            analysingCalculation &&
+            !!queue.find(
+                calculation =>
+                    calculation.endTimestamp != null && calculation.startTimestamp > analysingCalculation.startTimestamp
+            )
+        );
+    }
+
+    printCalculationsWaitingMoreThanSpecifiedSeconds(waitingLastsMs = 2000) {
+        const calculations = Object.keys(this._calculations)
+            .map(domain => this._calculations[domain].map(c => ({ ...c, domain })))
+            .flat()
+            .filter(c => c.endTimestamp === null && Date.now() - c.startTimestamp > waitingLastsMs);
+        Logger.log(
+            "printCalculationsWaitingMoreThanSpecifiedSeconds",
+            `Calculations waiting more than ${(waitingLastsMs / 1000).toFixed(1)}s:\n` +
+                calculations.map(c => `${c.domain}.${c.uuid.slice(0, 8)}: ${Date.now() - c.startTimestamp}\n`)
+        );
+    }
+}
+
+export const concurrentCalculationsMetadataHolder = new ConcurrentCalculationsMetadataHolder();

package/src/robust-api-caller/externalApiProvider.js

@@ -0,0 +1,156 @@
+export class ExternalApiProvider {
+    /**
+     * Creates an instance of external api provider.
+     *
+     * If you need sub-request then use 'subRequestIndex' to check current request index in functions below.
+     * Also use array for 'httpMethod'.
+     *
+     * If the endpoint of dedicated provider has pagination then you should customize the behavior using
+     * "changeQueryParametersForPageNumber", "checkWhetherResponseIsForLastPage".
+     *
+     * We perform RPS counting all over the App to avoid blocking our clients due to abuses of the providers.
+     *
+     * @param endpoint {string} URL to the provider's endpoint. Note: you can customize it using composeQueryString
+     * @param [httpMethod] {string|string[]} one of "get", "post", "put", "patch", "delete" or an array of these values
+     *        for request having sub-requests
+     * @param [timeout] {number} number of milliseconds to wait for the response
+     * @param [apiGroup] {ApiGroup} singleton object containing parameters of API group. Helpful when you use the same
+     *        api for different providers to avoid hardcoding RPS inside each provider what can cause mistakes
+     * @param [specificHeaders] {Object} contains specific keys (headers) and values (their content) if needed for this provider
+     * @param [maxPageLength] {number} optional number of items per page if the request supports pagination
+     */
+    constructor(
+        endpoint,
+        httpMethod,
+        timeout,
+        apiGroup,
+        specificHeaders = {},
+        maxPageLength = Number.MAX_SAFE_INTEGER
+    ) {
+        this.endpoint = endpoint;
+        this.httpMethod = httpMethod ?? "get";
+        // TODO: [refactoring, critical] We have two timeouts for robust data retrieval - here and inside the service method call, need to remain the only
+        this.timeout = timeout ?? 10000;
+        // TODO: [refactoring, critical] We need single place for all RPSes as we use them as hardcoded constants now inside different services
+        this.apiGroup = apiGroup;
+        this.maxPageLength = maxPageLength ?? Number.MAX_SAFE_INTEGER;
+        this.niceFactor = 1;
+        this.specificHeaders = specificHeaders ?? {};
+    }
+
+    getRps() {
+        return this.apiGroup.rps ?? 2;
+    }
+
+    isRpsExceeded() {
+        return this.apiGroup.isRpsExceeded();
+    }
+
+    actualizeLastCalledTimestamp() {
+        this.apiGroup.actualizeLastCalledTimestamp();
+    }
+
+    getApiGroupId() {
+        return this.apiGroup.id;
+    }
+
+    /**
+     * Some endpoint can require several sub requests. Example is one request to get confirmed transactions
+     * and another request for unconfirmed transactions. You should override this method to return true for such requests.
+     *
+     * @return {boolean} true if this provider requires several requests to retrieve the data
+     */
+    doesRequireSubRequests() {
+        return false;
+    }
+
+    /**
+     * Some endpoint support pagination. Override this method if so and implement corresponding methods.
+     *
+     * @return {boolean} true if this provider requires several requests to retrieve the data
+     */
+    doesSupportPagination() {
+        return false;
+    }
+
+    /**
+     * Composes a query string to be added to the endpoint of this provider.
+     *
+     * @param params {any[]} params array passed to the RobustExternalAPICallerService
+     * @param [subRequestIndex] {number} optional number of the sub-request the call is performed for
+     * @returns {string} query string to be concatenated with endpoint
+     */
+    composeQueryString(params, subRequestIndex = 0) {
+        return "";
+    }
+
+    /**
+     * Composes a body to be added to the request
+     *
+     * @param params {any[]} params array passed to the RobustExternalAPICallerService
+     * @param [subRequestIndex] {number} optional number of the sub-request the call is performed for
+     * @returns {string}
+     */
+    composeBody(params, subRequestIndex = 0) {
+        return "";
+    }
+
+    /**
+     * Extracts data from the response and returns it
+     *
+     * @param response {Object} HTTP response returned by provider
+     * @param [params] {any[]} params array passed to the RobustExternalAPICallerService
+     * @param [subRequestIndex] {number} optional number of the sub-request the call is performed for
+     * @param iterationsData {any[]} array of data retrieved from previous sub-requests
+     * @returns {any}
+     */
+    getDataByResponse(response, params = [], subRequestIndex = 0, iterationsData = []) {
+        return [];
+    }
+
+    /**
+     * Function changing the query string according to page number and previous response
+     * Only for endpoints supporting pagination
+     *
+     * @param params {any[]} params array passed to the RobustExternalAPICallerService
+     * @param previousResponse {Object} HTTP response returned by provider for previous call (previous page)
+     * @param pageNumber {number} new page number. We count from 0. You need to manually increment with 1 if your
+     *        provider counts pages starting with 1
+     * @param [subRequestIndex] {number} optional number of the sub-request the call is performed for
+     * @returns {any[]}
+     */
+    changeQueryParametersForPageNumber(params, previousResponse, pageNumber, subRequestIndex = 0) {
+        return params;
+    }
+
+    /**
+     * Function checking whether the response is for the last page to stop requesting for a next page.
+     * Only for endpoints supporting pagination.
+     *
+     * @param previousResponse {Object} HTTP response returned by provider for previous call (previous page)
+     * @param currentResponse {Object} HTTP response returned by provider for current call (current page, next after the previous)
+     * @param currentPageNumber {number} current page number (for current response)
+     * @param [subRequestIndex] {number} optional number of the sub-request the call is performed for
+     * @returns {boolean}
+     */
+    checkWhetherResponseIsForLastPage(previousResponse, currentResponse, currentPageNumber, subRequestIndex = 0) {
+        return true;
+    }
+
+    /**
+     * Resets the nice factor to default value
+     */
+    resetNiceFactor() {
+        this.niceFactor = 1;
+    }
+
+    /**
+     * Internal method used for requests requiring sub-requests.
+     *
+     * @param iterationsData {any[]} iterations data retrieved from getDataByResponse called per sub-request.
+     * @return {any} by default flatten the passed iterations data array. Should be redefined if you need another logic.
+     */
+    incorporateIterationsData(iterationsData) {
+        return iterationsData.flat();
+    }
+}

package/src/robust-api-caller/externalServicesStatsCollector.js

@@ -0,0 +1,78 @@
+import { improveAndRethrow } from "../common-apis/utils/errorUtils.js";
+import { Logger } from "../common-apis/utils/logging/logger.js";
+
+export class ExternalServicesStatsCollector {
+    constructor() {
+        this.stats = new Map();
+    }
+
+    externalServiceFailed(serviceUrl, message) {
+        try {
+            const processMessage = (stat, errorMessage) => {
+                const errors = stat.errors ?? {};
+                errorMessage = errorMessage ?? "";
+                if (errorMessage.match(/.*network.+error.*/i)) {
+                    errors["networkError"] = (errors["networkError"] || 0) + 1;
+                } else if (errorMessage.match(/.*timeout.+exceeded.*/i)) {
+                    errors["timeoutExceeded"] = (errors["timeoutExceeded"] || 0) + 1;
+                } else if (errors["other"]) {
+                    errors["other"].push(message);
+                } else {
+                    errors["other"] = [message];
+                }
+
+                stat.errors = errors;
+            };
+
+            if (this.stats.has(serviceUrl)) {
+                const stat = this.stats.get(serviceUrl);
+                stat.callsCount += 1;
+                stat.failsCount += 1;
+                processMessage(stat, message);
+            } else {
+                this.stats.set(serviceUrl, { callsCount: 1, failsCount: 1 });
+                processMessage(this.stats.get(serviceUrl), message);
+            }
+        } catch (e) {
+            improveAndRethrow(e, "externalServiceFailed");
+        }
+    }
+
+    externalServiceCalledWithoutError(serviceUrl) {
+        try {
+            if (this.stats.has(serviceUrl)) {
+                const stat = this.stats.get(serviceUrl);
+                stat.callsCount += 1;
+            } else {
+                this.stats.set(serviceUrl, { callsCount: 1, failsCount: 0 });
+            }
+        } catch (e) {
+            improveAndRethrow(e, "externalServiceCalledWithoutError");
+        }
+    }
+
+    /**
+     * Returns statistics about external services failures.
+     * Provides how many calls were performed and what the percent of failed calls. Also returns errors stat.
+     *
+     * @return {Array<object>} Array of objects of type { failsPerCent: number, calls: number }
+     *                         sorted by the highest fails percent desc
+     */
+    getStats() {
+        try {
+            return Array.from(this.stats.keys())
+                .map(key => {
+                    const stat = this.stats.get(key);
+                    return {
+                        url: key,
+                        failsPerCent: ((stat.failsCount / stat.callsCount) * 100).toFixed(2),
+                        calls: stat.callsCount,
+                        errors: stat.errors ?? [],
+                    };
+                })
+                .sort((s1, s2) => s1.failsPerCent - s2.failsPerCent);
+        } catch (e) {
+            Logger.logError(e, "getStats");
+        }
+    }
+}

package/src/robust-api-caller/robustExternalAPICallerService.js

@@ -0,0 +1,304 @@
+import { improveAndRethrow } from "../common-apis/utils/errorUtils.js";
+import { safeStringify } from "../common-apis/utils/safeStringify.js";
+import { Logger } from "../common-apis/utils/logging/logger.js";
+import { postponeExecution } from "../common-apis/utils/postponeExecution.js";
+import { AxiosAdapter } from "../common-apis/adapters/axiosAdapter.js";
+
+import { concurrentCalculationsMetadataHolder } from "./concurrentCalculationsMetadataHolder.js";
+import { ExternalServicesStatsCollector } from "./externalServicesStatsCollector.js";
+
+/**
+ * TODO: [refactoring, critical] update backend copy of this service. Also there is a task to extract this
+ *                               service and other related to it stuff to dedicated npm package task_id=b008ee5e4a3f42c08c73831c4bb3db4e
+ *
+ * Template service needed to avoid duplication of the same logic when we need to call
+ * external APIs to retrieve some data. The idea is to use several API providers to retrieve the same data. It helps to
+ * improve the reliability of a data retrieval.
+ */
+export class RobustExternalAPICallerService {
+    static statsCollector = new ExternalServicesStatsCollector();
+
+    static getStats() {
+        return this.statsCollector.getStats();
+    }
+
+    /**
+     * @param bio {string} service name for logging
+     * @param providersData {ExternalApiProvider[]} array of providers
+     * @param [logger] {function} function to be used for logging
+     */
+    constructor(bio, providersData, logger = Logger.logError) {
+        providersData.forEach(provider => {
+            if ((!provider.endpoint && provider.endpoint !== "") || !provider.httpMethod) {
+                throw new Error(`Wrong format of providers data for: ${JSON.stringify(provider)}`);
+            }
+        });
+
+        // We add niceFactor - just number to order the providers array by. It is helpful to call
+        // less robust APIs only if more robust fails
+        this.providers = providersData;
+        providersData.forEach(provider => provider.resetNiceFactor());
+        this.bio = bio;
+        this._logger = (...args) => {
+            Logger.logError(...args);
+        };
+    }
+
+    static defaultRPSFactor = 1;
+    static rpsMultiplier = 1.05;
+
+    /**
+     * Performs data retrieval from external APIs. Tries providers till the data is retrieved.
+     *
+     * @param parametersValues {array} array of values of the parameters for URL query string [and/or body]
+     * @param timeoutMS {number} http timeout to wait for response. If provider has its specific timeout value then it is used
+     * @param [cancelToken] {object|undefined} axios token to force-cancel requests from high-level code
+     * @param [attemptsCount] {number|undefined} number of attempts to be performed
+     * @param [doNotFailForNowData] {boolean|undefined} pass true if you do not want us to throw an error if we retrieved null data from all the providers
+     * @return {Promise<any>} resolving to retrieved data (or array of results if specific provider requires
+     *         several requests. NOTE: we flatten nested arrays - results of each separate request done for the specific provider)
+     * @throws Error if requests to all providers are failed
+     */
+    async callExternalAPI(
+        parametersValues = [],
+        timeoutMS = 3500,
+        cancelToken = null,
+        attemptsCount = 1,
+        doNotFailForNowData = false
+    ) {
+        let result;
+        const calculationUuid = concurrentCalculationsMetadataHolder.startCalculation(this.bio);
+
+        try {
+            for (let i = 0; (i < attemptsCount || result?.shouldBeForceRetried) && result?.data == null; ++i) {
+                /**
+                 * We use rpsFactor to improve re-attempting to call the providers if the last attempt resulted with
+                 * the fail due to abused RPSes of some (most part of) providers.
+                 * The _performCallAttempt in such a case will return increased rpsFactor inside the result object.
+                 */
+                const rpsFactor = result ? result.rpsFactor : RobustExternalAPICallerService.defaultRPSFactor;
+
+                result = null;
+
+                try {
+                    if (i === 0 && !result?.shouldBeForceRetried) {
+                        result = await this._performCallAttempt(
+                            parametersValues,
+                            timeoutMS,
+                            cancelToken,
+                            rpsFactor,
+                            doNotFailForNowData
+                        );
+                    } else {
+                        const maxRps = Math.max(...this.providers.map(provider => provider.getRps() ?? 0));
+                        const waitingTimeMs = maxRps ? 1000 / (maxRps / rpsFactor) : 0;
+
+                        result = await new Promise((resolve, reject) => {
+                            setTimeout(async () => {
+                                try {
+                                    resolve(
+                                        await this._performCallAttempt(
+                                            parametersValues,
+                                            timeoutMS,
+                                            cancelToken,
+                                            rpsFactor,
+                                            doNotFailForNowData
+                                        )
+                                    );
+                                } catch (e) {
+                                    reject(e);
+                                }
+                            }, waitingTimeMs);
+                        });
+                    }
+                    if (result.errors?.length) {
+                        const errors = result.errors;
+                        this._logger(
+                            new Error(
+                                `Failed at attempt ${i}. ${errors.length} errors. Messages: ${safeStringify(
+                                    errors.map(error => error.message)
+                                )}: ${safeStringify(errors)}.`
+                            ),
+                            `${this.bio}.callExternalAPI`,
+                            "",
+                            true
+                        );
+                    }
+                } catch (e) {
+                    this._logger(e, `${this.bio}.callExternalAPI`, "Failed to perform external providers calling");
+                }
+            }
+
+            if (result?.data == null) {
+                // TODO: [feature, moderate] looks like we should not fail for null data as it is strange - the provider will fail when processing data internally
+                const error = new Error(
+                    `Failed to retrieve data. It means all attempts have been failed. DEV: add more attempts to this data retrieval`
+                );
+                if (!doNotFailForNowData) {
+                    throw error;
+                } else {
+                    this._logger(error, `${this.bio}.callExternalAPI`);
+                }
+            }
+
+            return result?.data;
+        } catch (e) {
+            improveAndRethrow(e, `${this.bio}.callExternalAPI`);
+        } finally {
+            concurrentCalculationsMetadataHolder.endCalculation(this.bio, calculationUuid);
+        }
+    }
+
+    async _performCallAttempt(parametersValues, timeoutMS, cancelToken, rpsFactor, doNotFailForNowData) {
+        const providers = this._reorderProvidersByNiceFactor();
+        let data = undefined,
+            providerIndex = 0,
+            countOfRequestsDeclinedByRps = 0,
+            errors = [];
+        while (!data && providerIndex < providers.length) {
+            let provider = providers[providerIndex];
+            if (provider.isRpsExceeded()) {
+                /**
+                 * Current provider's RPS is exceeded, so we try next provider. Also, we count such cases to make
+                 * a decision about the force-retry need.
+                 */
+                ++providerIndex;
+                ++countOfRequestsDeclinedByRps;
+                continue;
+            }
+
+            try {
+                const axiosConfig = {
+                    ...(cancelToken ? { cancelToken } : {}),
+                    timeout: provider.timeout || timeoutMS,
+                    headers: provider.specificHeaders ?? {},
+                };
+                const httpMethods = Array.isArray(provider.httpMethod) ? provider.httpMethod : [provider.httpMethod];
+                const iterationsData = [];
+                for (let subRequestIndex = 0; subRequestIndex < httpMethods.length; ++subRequestIndex) {
+                    const query = provider.composeQueryString(parametersValues, subRequestIndex);
+                    const endpoint = `${provider.endpoint}${query}`;
+                    const axiosParams = [endpoint, axiosConfig];
+                    if (["post", "put", "patch"].find(method => method === httpMethods[subRequestIndex])) {
+                        const body = provider.composeBody(parametersValues, subRequestIndex) ?? null;
+                        axiosParams.splice(1, 0, body);
+                    }
+
+                    let pageNumber = 0;
+                    const responsesForPages = [];
+                    let hasNextPage = provider.doesSupportPagination();
+                    do {
+                        if (subRequestIndex === 0 && pageNumber === 0) {
+                            provider.actualizeLastCalledTimestamp();
+                            responsesForPages[pageNumber] = await AxiosAdapter.call(
+                                httpMethods[subRequestIndex],
+                                ...axiosParams
+                            );
+                            RobustExternalAPICallerService.statsCollector.externalServiceCalledWithoutError(
+                                provider.getApiGroupId()
+                            );
+                        } else {
+                            if (pageNumber > 0) {
+                                const actualizedParams = provider.changeQueryParametersForPageNumber(
+                                    parametersValues,
+                                    responsesForPages[pageNumber - 1],
+                                    pageNumber,
+                                    subRequestIndex
+                                );
+                                const query = provider.composeQueryString(actualizedParams, subRequestIndex);
+                                axiosParams[0] = `${provider.endpoint}${query}`;
+                            }
+                            /**
+                             * For second and more request we postpone each request to not exceed RPS
+                             * of current provider. We use rpsFactor to dynamically increase the rps to avoid
+                             * too frequent calls if we continue failing to retrieve the data due to RPS exceeding.
+                             * TODO: [dev] test RPS factor logic (units or integration)
+                             */
+
+                            const waitingTimeMS = provider.getRps() ? 1000 / (provider.getRps() / rpsFactor) : 0;
+
+                            const postponeUntilRpsExceeded = async (recursionLevel = 0) => {
+                                return await postponeExecution(async () => {
+                                    const maxCountOfPostponingAttempts = 2;
+                                    if (provider.isRpsExceeded() && recursionLevel < maxCountOfPostponingAttempts) {
+                                        return await postponeUntilRpsExceeded(recursionLevel + 1);
+                                    }
+                                    provider.actualizeLastCalledTimestamp();
+                                    return await AxiosAdapter.call(httpMethods[subRequestIndex], ...axiosParams);
+                                }, waitingTimeMS);
+                            };
+
+                            responsesForPages[pageNumber] = await postponeUntilRpsExceeded();
+                        }
+
+                        if (hasNextPage) {
+                            hasNextPage = !provider.checkWhetherResponseIsForLastPage(
+                                responsesForPages[pageNumber - 1],
+                                responsesForPages[pageNumber],
+                                pageNumber,
+                                subRequestIndex
+                            );
+                        }
+                        pageNumber++;
+                    } while (hasNextPage);
+
+                    const responsesDataForPages = responsesForPages.map(response =>
+                        provider.getDataByResponse(response, parametersValues, subRequestIndex, iterationsData)
+                    );
+
+                    let allData = responsesDataForPages;
+                    if (Array.isArray(responsesDataForPages[0])) {
+                        allData = responsesDataForPages.flat();
+                    } else if (responsesDataForPages.length === 1) {
+                        allData = responsesDataForPages[0];
+                    }
+
+                    iterationsData.push(allData);
+                }
+                if (iterationsData.length) {
+                    if (httpMethods.length > 1) {
+                        data = provider.incorporateIterationsData(iterationsData);
+                    } else {
+                        data = iterationsData[0];
+                    }
+                } else if (!doNotFailForNowData) {
+                    RobustExternalAPICallerService.statsCollector.externalServiceFailed(
+                        provider.getApiGroupId(),
+                        "Response data was null for some reason"
+                    );
+                    punishProvider(provider);
+                }
+            } catch (e) {
+                punishProvider(provider);
+                RobustExternalAPICallerService.statsCollector.externalServiceFailed(
+                    provider.getApiGroupId(),
+                    e?.message
+                );
+                errors.push(e);
+            } finally {
+                providerIndex++;
+            }
+        }
+
+        // If we are declining more than 50% of providers (by exceeding RPS) then we note that it better to retry the whole process of providers requesting
+        const shouldBeForceRetried = data == null && countOfRequestsDeclinedByRps > Math.floor(providers.length * 0.5);
+        const rpsMultiplier = shouldBeForceRetried ? RobustExternalAPICallerService.rpsMultiplier : 1;
+
+        return {
+            data: data ?? null,
+            shouldBeForceRetried,
+            rpsFactor: rpsFactor * rpsMultiplier,
+            errors,
+        };
+    }
+
+    _reorderProvidersByNiceFactor() {
+        const providersCopy = [...this.providers];
+
+        return providersCopy.sort((p1, p2) => p2.niceFactor - p1.niceFactor);
+    }
+}
+
+function punishProvider(provider) {
+    provider.niceFactor = provider.niceFactor - 1;
+}