@rabbitio/ui-kit 1.0.0-beta.37 → 1.0.0-beta.39

package/src/robustExteranlApiCallerService/cachedRobustExternalApiCallerService.js

@@ -0,0 +1,188 @@
+import Hashes from "jshashes";
+
+import { improveAndRethrow } from "../common/errorUtils.js";
+import { safeStringify } from "../common/utils/safeStringify.js";
+import { Logger } from "../common/utils/logging/logger.js";
+
+import { RobustExternalAPICallerService } from "./robustExternalAPICallerService.js";
+import { CacheAndConcurrentRequestsResolver } from "./cacheAndConcurrentRequestsResolver.js";
+
+/**
+ * Extended edit of RobustExternalApiCallerService supporting cache and management of concurrent requests
+ * to the same resource.
+ * TODO: [tests, critical] Massively used logic
+ */
+export class CachedRobustExternalApiCallerService {
+    /**
+     * @param bio {string} unique service identifier
+     * @param cache {Cache} cache instance
+     * @param providersData {ExternalApiProvider[]} array of providers
+     * @param [cacheTtlMs=10000] {number} time to live for cache ms
+     * @param [maxCallAttemptsToWaitForAlreadyRunningRequest=50] {number} see details in CacheAndConcurrentRequestsResolver
+     * @param [timeoutBetweenAttemptsToCheckWhetherAlreadyRunningRequestFinished=3000] {number} see details in CacheAndConcurrentRequestsResolver
+     * @param [removeExpiredCacheAutomatically=true] {boolean} whether to remove cached data automatically when ttl exceeds
+     * @param [mergeCachedAndNewlyRetrievedData=null] {function} function accepting cached data, newly retrieved data and id field name for list items
+     *        and merging them. use if needed
+     */
+    constructor(
+        bio,
+        cache,
+        providersData,
+        cacheTtlMs = 10000,
+        removeExpiredCacheAutomatically = true,
+        mergeCachedAndNewlyRetrievedData = null,
+        maxCallAttemptsToWaitForAlreadyRunningRequest = 100,
+        timeoutBetweenAttemptsToCheckWhetherAlreadyRunningRequestFinished = 1000
+    ) {
+        this._provider = new RobustExternalAPICallerService(
+            `cached_${bio}`,
+            providersData,
+            Logger.logError
+        );
+        this._cacheTtlMs = cacheTtlMs;
+        this._cahceAndRequestsResolver = new CacheAndConcurrentRequestsResolver(
+            bio,
+            cache,
+            cacheTtlMs,
+            removeExpiredCacheAutomatically,
+            maxCallAttemptsToWaitForAlreadyRunningRequest,
+            timeoutBetweenAttemptsToCheckWhetherAlreadyRunningRequestFinished
+        );
+        this._cahceIds = [];
+        this._mergeCachedAndNewlyRetrievedData =
+            mergeCachedAndNewlyRetrievedData;
+    }
+
+    /**
+     * Calls the external API or returns data from cache. Just waits if the same data already requested.
+     *
+     * @param parametersValues {array} array of values of the parameters for URL query string [and/or body]
+     * @param timeoutMS {number} http timeout to wait for response. If provider has its specific timeout value then it is used
+     * @param [cancelToken] {object|undefined} axios token to force-cancel requests from high-level code
+     * @param [attemptsCount] {number|undefined} number of attempts to be performed
+     * @param [customHashFunctionForParams] {function|undefined} function without params calculating the hash to be
+     *        added to bio of the service to compose a unique parameters-specific cache id
+     * @param [doNotFailForNowData] {boolean|undefined} pass true if you do not want us to throw an error if we retrieved null data from all the providers
+     * @return {Promise<any>} resolving to retrieved data (or array of results if specific provider requires
+     *         several requests. NOTE: we flatten nested arrays - results of each separate request done for the specific provider)
+     * @throws Error if requests to all providers are failed
+     */
+    async callExternalAPICached(
+        parametersValues = [],
+        timeoutMS = 3500,
+        cancelToken = null,
+        attemptsCount = 1,
+        customHashFunctionForParams = null,
+        doNotFailForNowData = false
+    ) {
+        const loggerSource = `${this._provider.bio}.callExternalAPICached`;
+        let cacheId;
+        let result;
+        try {
+            cacheId = this._calculateCacheId(
+                parametersValues,
+                customHashFunctionForParams
+            );
+            result =
+                await this._cahceAndRequestsResolver.getCachedOrWaitForCachedOrAcquireLock(
+                    cacheId
+                );
+            if (!result?.canStartDataRetrieval) {
+                return result?.cachedData;
+            }
+
+            let data = await this._provider.callExternalAPI(
+                parametersValues,
+                timeoutMS,
+                cancelToken,
+                attemptsCount,
+                doNotFailForNowData
+            );
+
+            const canPerformMerge =
+                typeof this._mergeCachedAndNewlyRetrievedData === "function";
+            if (canPerformMerge) {
+                const mostRecentCached =
+                    this._cahceAndRequestsResolver.getCached(cacheId);
+                data = this._mergeCachedAndNewlyRetrievedData(
+                    mostRecentCached,
+                    data,
+                    parametersValues
+                );
+            }
+            if (data != null) {
+                this._cahceAndRequestsResolver.saveCachedData(
+                    cacheId,
+                    result?.lockId,
+                    data,
+                    true,
+                    canPerformMerge
+                );
+                this._cahceIds.indexOf(cacheId) < 0 &&
+                    this._cahceIds.push(cacheId);
+            }
+
+            return data;
+        } catch (e) {
+            improveAndRethrow(e, loggerSource);
+        } finally {
+            this._cahceAndRequestsResolver.releaseLock(cacheId, result?.lockId);
+        }
+    }
+
+    invalidateCaches() {
+        this._cahceIds.forEach((key) =>
+            this._cahceAndRequestsResolver.invalidate(key)
+        );
+    }
+
+    actualizeCachedData(
+        params,
+        synchronousCurrentCacheProcessor,
+        customHashFunctionForParams = null,
+        sessionDependent = true,
+        actualizedAtTimestamp
+    ) {
+        const cacheId = this._calculateCacheId(
+            params,
+            customHashFunctionForParams
+        );
+        this._cahceAndRequestsResolver.actualizeCachedData(
+            cacheId,
+            synchronousCurrentCacheProcessor,
+            sessionDependent
+        );
+    }
+
+    markCacheAsExpiredButDontRemove(
+        parametersValues,
+        customHashFunctionForParams
+    ) {
+        try {
+            this._cahceAndRequestsResolver.markAsExpiredButDontRemove(
+                this._calculateCacheId(
+                    parametersValues,
+                    customHashFunctionForParams
+                )
+            );
+        } catch (e) {
+            improveAndRethrow(e, "markCacheAsExpiredButDontRemove");
+        }
+    }
+
+    _calculateCacheId(parametersValues, customHashFunctionForParams = null) {
+        try {
+            const hash =
+                typeof customHashFunctionForParams === "function"
+                    ? customHashFunctionForParams(parametersValues)
+                    : !parametersValues
+                      ? ""
+                      : new Hashes.SHA512().hex(
+                            safeStringify(parametersValues)
+                        );
+            return `${this._provider.bio}-${hash}`;
+        } catch (e) {
+            improveAndRethrow(e, this._provider.bio + "_calculateCacheId");
+        }
+    }
+}

package/src/robustExteranlApiCallerService/cancelProcessing.js

@@ -0,0 +1,29 @@
+import axios from "axios";
+
+/**
+ * Utils class needed to perform cancelling of axios request inside some process.
+ * Provides cancel state and axios token for HTTP requests
+ */
+export class CancelProcessing {
+    constructor() {
+        this._cancelToken = axios.CancelToken.source();
+        this._isCanceled = false;
+    }
+
+    cancel() {
+        this._isCanceled = true;
+        this._cancelToken.cancel();
+    }
+
+    isCanceled() {
+        return this._isCanceled;
+    }
+
+    getToken() {
+        return this._cancelToken.token;
+    }
+
+    static instance() {
+        return new CancelProcessing();
+    }
+}

package/src/robustExteranlApiCallerService/concurrentCalculationsMetadataHolder.js

@@ -0,0 +1,103 @@
+import { v4 } from "uuid";
+
+import { Logger } from "../common/utils/logging/logger.js";
+
+// TODO: [refactoring, low] Consider removing this logic task_id=c360f2af75764bde8badd9ff1cc00d48
+class ConcurrentCalculationsMetadataHolder {
+    constructor() {
+        this._calculations = {};
+    }
+
+    startCalculation(domain, calculationsHistoryMaxLength = 100) {
+        if (!this._calculations[domain]) {
+            this._calculations[domain] = [];
+        }
+
+        if (this._calculations[domain].length > calculationsHistoryMaxLength) {
+            this._calculations[domain] = this._calculations[domain].slice(
+                Math.round(calculationsHistoryMaxLength * 0.2)
+            );
+        }
+
+        const newCalculation = {
+            startTimestamp: Date.now(),
+            endTimestamp: null,
+            uuid: v4(),
+        };
+
+        this._calculations[domain].push(newCalculation);
+
+        return newCalculation.uuid;
+    }
+
+    endCalculation(domain, uuid, isFailed = false) {
+        try {
+            const calculation = this._calculations[domain].find(
+                (calculation) => calculation?.uuid === uuid
+            );
+            if (calculation) {
+                calculation.endTimestamp = Date.now();
+                calculation.isFiled = isFailed;
+            }
+
+            const elapsed = (
+                ((calculation?.endTimestamp ?? 0) -
+                    (calculation?.startTimestamp ?? 0)) /
+                1000
+            ).toFixed(1);
+            Logger.log(
+                "endCalculation",
+                `${elapsed} ms: ${domain}.${(calculation?.uuid ?? "").slice(
+                    0,
+                    7
+                )}`
+            );
+
+            return calculation;
+        } catch (e) {
+            Logger.logError(e, "endCalculation");
+        }
+    }
+
+    isCalculationLate(domain, uuid) {
+        const queue = this._calculations[domain];
+        const analysingCalculation = queue.find((item) => item.uuid === uuid);
+        return (
+            analysingCalculation &&
+            !!queue.find(
+                (calculation) =>
+                    calculation.endTimestamp != null &&
+                    calculation.startTimestamp >
+                        analysingCalculation.startTimestamp
+            )
+        );
+    }
+
+    printCalculationsWaitingMoreThanSpecifiedSeconds(waitingLastsMs = 2000) {
+        const calculations = Object.keys(this._calculations)
+            .map((domain) =>
+                this._calculations[domain].map((c) => ({ ...c, domain }))
+            )
+            .flat()
+            .filter(
+                (c) =>
+                    c.endTimestamp === null &&
+                    Date.now() - c.startTimestamp > waitingLastsMs
+            );
+        Logger.log(
+            "printCalculationsWaitingMoreThanSpecifiedSeconds",
+            `Calculations waiting more than ${(waitingLastsMs / 1000).toFixed(
+                1
+            )}s:\n` +
+                calculations.map(
+                    (c) =>
+                        `${c.domain}.${c.uuid.slice(0, 8)}: ${
+                            Date.now() - c.startTimestamp
+                        }\n`
+                )
+        );
+    }
+}
+
+export const concurrentCalculationsMetadataHolder =
+    new ConcurrentCalculationsMetadataHolder();

package/src/robustExteranlApiCallerService/externalApiProvider.js

@@ -0,0 +1,156 @@
+export class ExternalApiProvider {
+    /**
+     * Creates an instance of external api provider.
+     *
+     * If you need sub-request then use 'subRequestIndex' to check current request index in functions below.
+     * Also use array for 'httpMethod'.
+     *
+     * If the endpoint of dedicated provider has pagination then you should customize the behavior using
+     * "changeQueryParametersForPageNumber", "checkWhetherResponseIsForLastPage".
+     *
+     * We perform RPS counting all over the App to avoid blocking our clients due to abuses of the providers.
+     *
+     * @param endpoint {string} URL to the provider's endpoint. Note: you can customize it using composeQueryString
+     * @param [httpMethod] {string|string[]} one of "get", "post", "put", "patch", "delete" or an array of these values
+     *        for request having sub-requests
+     * @param [timeout] {number} number of milliseconds to wait for the response
+     * @param [apiGroup] {ApiGroup} singleton object containing parameters of API group. Helpful when you use the same
+     *        api for different providers to avoid hardcoding RPS inside each provider what can cause mistakes
+     * @param [specificHeaders] {Object} contains specific keys (headers) and values (their content) if needed for this provider
+     * @param [maxPageLength] {number} optional number of items per page if the request supports pagination
+     */
+    constructor(
+        endpoint,
+        httpMethod,
+        timeout,
+        apiGroup,
+        specificHeaders = {},
+        maxPageLength = Number.MAX_SAFE_INTEGER
+    ) {
+        this.endpoint = endpoint;
+        this.httpMethod = httpMethod ?? "get";
+        // TODO: [refactoring, critical] We have two timeouts for robust data retrieval - here and inside the service method call, need to remain the only
+        this.timeout = timeout ?? 10000;
+        // TODO: [refactoring, critical] We need single place for all RPSes as we use them as hardcoded constants now inside different services
+        this.apiGroup = apiGroup;
+        this.maxPageLength = maxPageLength ?? Number.MAX_SAFE_INTEGER;
+        this.niceFactor = 1;
+        this.specificHeaders = specificHeaders ?? {};
+    }
+
+    getRps() {
+        return this.apiGroup.rps ?? 2;
+    }
+
+    isRpsExceeded() {
+        return this.apiGroup.isRpsExceeded();
+    }
+
+    actualizeLastCalledTimestamp() {
+        this.apiGroup.actualizeLastCalledTimestamp();
+    }
+
+    getApiGroupId() {
+        return this.apiGroup.id;
+    }
+
+    /**
+     * Some endpoint can require several sub requests. Example is one request to get confirmed transactions
+     * and another request for unconfirmed transactions. You should override this method to return true for such requests.
+     *
+     * @return {boolean} true if this provider requires several requests to retrieve the data
+     */
+    doesRequireSubRequests() {
+        return false;
+    }
+
+    /**
+     * Some endpoint support pagination. Override this method if so and implement corresponding methods.
+     *
+     * @return {boolean} true if this provider requires several requests to retrieve the data
+     */
+    doesSupportPagination() {
+        return false;
+    }
+
+    /**
+     * Composes a query string to be added to the endpoint of this provider.
+     *
+     * @param params {any[]} params array passed to the RobustExternalAPICallerService
+     * @param [subRequestIndex] {number} optional number of the sub-request the call is performed for
+     * @returns {string} query string to be concatenated with endpoint
+     */
+    composeQueryString(params, subRequestIndex = 0) {
+        return "";
+    }
+
+    /**
+     * Composes a body to be added to the request
+     *
+     * @param params {any[]} params array passed to the RobustExternalAPICallerService
+     * @param [subRequestIndex] {number} optional number of the sub-request the call is performed for
+     * @returns {string}
+     */
+    composeBody(params, subRequestIndex = 0) {
+        return "";
+    }
+
+    /**
+     * Extracts data from the response and returns it
+     *
+     * @param response {Object} HTTP response returned by provider
+     * @param [params] {any[]} params array passed to the RobustExternalAPICallerService
+     * @param [subRequestIndex] {number} optional number of the sub-request the call is performed for
+     * @param iterationsData {any[]} array of data retrieved from previous sub-requests
+     * @returns {any}
+     */
+    getDataByResponse(response, params = [], subRequestIndex = 0, iterationsData = []) {
+        return [];
+    }
+
+    /**
+     * Function changing the query string according to page number and previous response
+     * Only for endpoints supporting pagination
+     *
+     * @param params {any[]} params array passed to the RobustExternalAPICallerService
+     * @param previousResponse {Object} HTTP response returned by provider for previous call (previous page)
+     * @param pageNumber {number} new page number. We count from 0. You need to manually increment with 1 if your
+     *        provider counts pages starting with 1
+     * @param [subRequestIndex] {number} optional number of the sub-request the call is performed for
+     * @returns {any[]}
+     */
+    changeQueryParametersForPageNumber(params, previousResponse, pageNumber, subRequestIndex = 0) {
+        return params;
+    }
+
+    /**
+     * Function checking whether the response is for the last page to stop requesting for a next page.
+     * Only for endpoints supporting pagination.
+     *
+     * @param previousResponse {Object} HTTP response returned by provider for previous call (previous page)
+     * @param currentResponse {Object} HTTP response returned by provider for current call (current page, next after the previous)
+     * @param currentPageNumber {number} current page number (for current response)
+     * @param [subRequestIndex] {number} optional number of the sub-request the call is performed for
+     * @returns {boolean}
+     */
+    checkWhetherResponseIsForLastPage(previousResponse, currentResponse, currentPageNumber, subRequestIndex = 0) {
+        return true;
+    }
+
+    /**
+     * Resets the nice factor to default value
+     */
+    resetNiceFactor() {
+        this.niceFactor = 1;
+    }
+
+    /**
+     * Internal method used for requests requiring sub-requests.
+     *
+     * @param iterationsData {any[]} iterations data retrieved from getDataByResponse called per sub-request.
+     * @return {any} by default flatten the passed iterations data array. Should be redefined if you need another logic.
+     */
+    incorporateIterationsData(iterationsData) {
+        return iterationsData.flat();
+    }
+}

package/src/robustExteranlApiCallerService/externalServicesStatsCollector.js

@@ -0,0 +1,82 @@
+import { improveAndRethrow } from "../common/errorUtils.js";
+import { Logger } from "../common/utils/logging/logger.js";
+
+export class ExternalServicesStatsCollector {
+    constructor() {
+        this.stats = new Map();
+    }
+
+    externalServiceFailed(serviceUrl, message) {
+        try {
+            const processMessage = (stat, errorMessage) => {
+                const errors = stat.errors ?? {};
+                errorMessage = errorMessage ?? "";
+                if (errorMessage.match(/.*network.+error.*/i)) {
+                    errors["networkError"] = (errors["networkError"] || 0) + 1;
+                } else if (errorMessage.match(/.*timeout.+exceeded.*/i)) {
+                    errors["timeoutExceeded"] =
+                        (errors["timeoutExceeded"] || 0) + 1;
+                } else if (errors["other"]) {
+                    errors["other"].push(message);
+                } else {
+                    errors["other"] = [message];
+                }
+
+                stat.errors = errors;
+            };
+
+            if (this.stats.has(serviceUrl)) {
+                const stat = this.stats.get(serviceUrl);
+                stat.callsCount += 1;
+                stat.failsCount += 1;
+                processMessage(stat, message);
+            } else {
+                this.stats.set(serviceUrl, { callsCount: 1, failsCount: 1 });
+                processMessage(this.stats.get(serviceUrl), message);
+            }
+        } catch (e) {
+            improveAndRethrow(e, "externalServiceFailed");
+        }
+    }
+
+    externalServiceCalledWithoutError(serviceUrl) {
+        try {
+            if (this.stats.has(serviceUrl)) {
+                const stat = this.stats.get(serviceUrl);
+                stat.callsCount += 1;
+            } else {
+                this.stats.set(serviceUrl, { callsCount: 1, failsCount: 0 });
+            }
+        } catch (e) {
+            improveAndRethrow(e, "externalServiceCalledWithoutError");
+        }
+    }
+
+    /**
+     * Returns statistics about external services failures.
+     * Provides how many calls were performed and what the percent of failed calls. Also returns errors stat.
+     *
+     * @return {Array<object>} Array of objects of type { failsPerCent: number, calls: number }
+     *                         sorted by the highest fails percent desc
+     */
+    getStats() {
+        try {
+            return Array.from(this.stats.keys())
+                .map((key) => {
+                    const stat = this.stats.get(key);
+                    return {
+                        url: key,
+                        failsPerCent: (
+                            (stat.failsCount / stat.callsCount) *
+                            100
+                        ).toFixed(2),
+                        calls: stat.callsCount,
+                        errors: stat.errors ?? [],
+                    };
+                })
+                .sort((s1, s2) => s1.failsPerCent - s2.failsPerCent);
+        } catch (e) {
+            Logger.logError(e, "getStats");
+        }
+    }
+}