@crowdin/app-project-module 0.30.1 → 0.30.3

package/out/util/api/api.d.ts

@@ -1,5 +1,5 @@
-import { ApiEndpoints, ApiModule, Config, CrowdinClientRequest, CrowdinContextInfo } from '../../models';
 import { Express } from 'express';
+import { ApiEndpoints, ApiModule, Config, CrowdinClientRequest, CrowdinContextInfo } from '../../models';
 export declare function getApiManifest(config: Config, apiModule: ApiModule): ApiEndpoints[];
 export declare function updateCrowdinContext(req: CrowdinClientRequest, context: CrowdinContextInfo): CrowdinContextInfo;
 export declare function addDefaultApiEndpoints(app: Express, config: Config): void;

package/out/util/api/api.js

@@ -4,23 +4,23 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.addSwagerApiDocumentation = exports.addDefaultApiEndpoints = exports.updateCrowdinContext = exports.getApiManifest = void 0;
-const models_1 = require("../../models");
+const path_1 = __importDefault(require("path"));
 const redoc_express_1 = __importDefault(require("redoc-express"));
 const swagger_jsdoc_1 = __importDefault(require("swagger-jsdoc"));
-const json_response_1 = __importDefault(require("../../middlewares/json-response"));
+const crowdin_file_progress_1 = __importDefault(require("../../handlers/integration/crowdin-file-progress"));
+const crowdin_files_1 = __importDefault(require("../../handlers/integration/crowdin-files"));
+const crowdin_update_1 = __importDefault(require("../../handlers/integration/crowdin-update"));
+const integration_data_1 = __importDefault(require("../../handlers/integration/integration-data"));
+const integration_login_1 = __importDefault(require("../../handlers/integration/integration-login"));
+const integration_update_1 = __importDefault(require("../../handlers/integration/integration-update"));
+const settings_1 = __importDefault(require("../../handlers/integration/settings"));
+const settings_save_1 = __importDefault(require("../../handlers/integration/settings-save"));
+const sync_settings_1 = __importDefault(require("../../handlers/integration/sync-settings"));
+const sync_settings_save_1 = __importDefault(require("../../handlers/integration/sync-settings-save"));
 const crowdin_client_1 = __importDefault(require("../../middlewares/crowdin-client"));
 const integration_credentials_1 = __importDefault(require("../../middlewares/integration-credentials"));
-const crowdin_files_1 = __importDefault(require("../../handlers/crowdin-files"));
-const integration_data_1 = __importDefault(require("../../handlers/integration-data"));
-const crowdin_file_progress_1 = __importDefault(require("../../handlers/crowdin-file-progress"));
-const crowdin_update_1 = __importDefault(require("../../handlers/crowdin-update"));
-const sync_settings_1 = __importDefault(require("../../handlers/sync-settings"));
-const sync_settings_save_1 = __importDefault(require("../../handlers/sync-settings-save"));
-const settings_save_1 = __importDefault(require("../../handlers/settings-save"));
-const settings_1 = __importDefault(require("../../handlers/settings"));
-const integration_update_1 = __importDefault(require("../../handlers/integration-update"));
-const path_1 = __importDefault(require("path"));
-const integration_login_1 = __importDefault(require("../../handlers/integration-login"));
+const json_response_1 = __importDefault(require("../../middlewares/json-response"));
+const models_1 = require("../../models");
 function getApiManifest(config, apiModule) {
     const apiModuleManifest = [];
     if (apiModule.endpoints) {

package/out/util/axios.d.ts

@@ -0,0 +1,10 @@
+import { AxiosInstance } from 'axios';
+export declare class AxiosProvider {
+    private static readonly CROWDIN_API_MAX_CONCURRENT_REQUESTS;
+    private static readonly CROWDIN_API_REQUESTS_INTERVAL_MS;
+    private pendingRequests;
+    axios: AxiosInstance;
+    constructor();
+    private configureRequest;
+    private configureResponse;
+}

package/out/util/axios.js

@@ -0,0 +1,42 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AxiosProvider = void 0;
+const axios_1 = __importDefault(require("axios"));
+//copy from api client to control throughput
+class AxiosProvider {
+    constructor() {
+        this.pendingRequests = 0;
+        this.axios = axios_1.default.create({});
+        this.configureRequest();
+        this.configureResponse();
+    }
+    configureRequest() {
+        this.axios.interceptors.request.use((config) => {
+            // eslint-disable-next-line @typescript-eslint/explicit-function-return-type
+            return new Promise((resolve) => {
+                const interval = setInterval(() => {
+                    if (this.pendingRequests < AxiosProvider.CROWDIN_API_MAX_CONCURRENT_REQUESTS) {
+                        this.pendingRequests++;
+                        clearInterval(interval);
+                        resolve(config);
+                    }
+                }, AxiosProvider.CROWDIN_API_REQUESTS_INTERVAL_MS);
+            });
+        });
+    }
+    configureResponse() {
+        this.axios.interceptors.response.use((response) => {
+            this.pendingRequests = Math.max(0, this.pendingRequests - 1);
+            return Promise.resolve(response);
+        }, (error) => {
+            this.pendingRequests = Math.max(0, this.pendingRequests - 1);
+            return Promise.reject(error);
+        });
+    }
+}
+exports.AxiosProvider = AxiosProvider;
+AxiosProvider.CROWDIN_API_MAX_CONCURRENT_REQUESTS = 15;
+AxiosProvider.CROWDIN_API_REQUESTS_INTERVAL_MS = 10;

package/out/util/connection.d.ts

@@ -1,6 +1,6 @@
 import Crowdin from '@crowdin/crowdin-api-client';
 import { AccountType, Config, CrowdinCredentials, IntegrationCredentials, IntegrationLogic, SubscriptionInfo } from '../models';
-export declare function prepareCrowdinClient(config: Config, credentials: CrowdinCredentials): Promise<{
+export declare function prepareCrowdinClient(config: Config, credentials: CrowdinCredentials, autoRenew?: boolean): Promise<{
     client: Crowdin;
     token: string;
 }>;

package/out/util/connection.js

@@ -19,6 +19,8 @@ const axios_1 = __importDefault(require("axios"));
 const _1 = require(".");
 const models_1 = require("../models");
 const storage_1 = require("../storage");
+const axios_2 = require("./axios");
+const axiosCustom = new axios_2.AxiosProvider().axios;
 function refreshToken(config, credentials) {
     var _a, _b;
     return __awaiter(this, void 0, void 0, function* () {
@@ -51,23 +53,163 @@ function refreshToken(config, credentials) {
         };
     });
 }
-function prepareCrowdinClient(config, credentials) {
+function prepareCrowdinClient(config, credentials, autoRenew = false) {
     var _a, _b;
     return __awaiter(this, void 0, void 0, function* () {
         //2 min as an extra buffer
         const isExpired = +credentials.expire - 120 < Date.now() / 1000;
         const organization = credentials.type === models_1.AccountType.ENTERPRISE ? credentials.id : undefined;
+        let token;
+        let crowdinCreds;
         if (!isExpired) {
-            const token = (0, _1.decryptData)(config, credentials.accessToken);
+            token = (0, _1.decryptData)(config, credentials.accessToken);
+            crowdinCreds = credentials;
+        }
+        else {
+            const res = yield refreshCrowdinCreds({ config, credentials });
+            token = res.token;
+            crowdinCreds = res.credentials;
+        }
+        if (!autoRenew) {
             return {
                 client: new crowdin_api_client_1.default({ token, organization, baseUrl: (_a = config.crowdinUrls) === null || _a === void 0 ? void 0 : _a.apiUrl }),
                 token,
             };
         }
+        const getHttpConfig = (config) => {
+            if (!config) {
+                return;
+            }
+            return { headers: Object.assign(Object.assign({}, config.headers), { Authorization: `Bearer ${token}` }) };
+        };
+        const renewToken = () => __awaiter(this, void 0, void 0, function* () {
+            const res = yield refreshCrowdinCreds({ config, credentials: crowdinCreds });
+            token = res.token;
+            crowdinCreds = res.credentials;
+        });
+        const isExpiredCause = (error) => {
+            var _a, _b;
+            const errorObj = (_b = (_a = error === null || error === void 0 ? void 0 : error.response) === null || _a === void 0 ? void 0 : _a.data) === null || _b === void 0 ? void 0 : _b.error;
+            return (errorObj === null || errorObj === void 0 ? void 0 : errorObj.code) === 401 && (errorObj === null || errorObj === void 0 ? void 0 : errorObj.message) === 'Unauthorized';
+        };
+        //providing custom http client to handle expirations and renew token on the fly
+        //needed for long running jobs with single Crowdin client instance
+        return {
+            client: new crowdin_api_client_1.default({
+                token,
+                organization,
+                baseUrl: (_b = config.crowdinUrls) === null || _b === void 0 ? void 0 : _b.apiUrl,
+            }, {
+                httpClient: {
+                    get(url, httpConfig) {
+                        return __awaiter(this, void 0, void 0, function* () {
+                            try {
+                                const res = yield axiosCustom.get(url, getHttpConfig(httpConfig));
+                                return res.data;
+                            }
+                            catch (e) {
+                                if (isExpiredCause(e)) {
+                                    yield renewToken();
+                                    const res = yield axiosCustom.get(url, getHttpConfig(httpConfig));
+                                    return res.data;
+                                }
+                                throw e;
+                            }
+                        });
+                    },
+                    delete(url, httpConfig) {
+                        return __awaiter(this, void 0, void 0, function* () {
+                            try {
+                                const res = yield axiosCustom.delete(url, getHttpConfig(httpConfig));
+                                return res.data;
+                            }
+                            catch (e) {
+                                if (isExpiredCause(e)) {
+                                    yield renewToken();
+                                    const res = yield axiosCustom.delete(url, getHttpConfig(httpConfig));
+                                    return res.data;
+                                }
+                                throw e;
+                            }
+                        });
+                    },
+                    head(url, httpConfig) {
+                        return __awaiter(this, void 0, void 0, function* () {
+                            try {
+                                const res = yield axiosCustom.head(url, getHttpConfig(httpConfig));
+                                return res.data;
+                            }
+                            catch (e) {
+                                if (isExpiredCause(e)) {
+                                    yield renewToken();
+                                    const res = yield axiosCustom.head(url, getHttpConfig(httpConfig));
+                                    return res.data;
+                                }
+                                throw e;
+                            }
+                        });
+                    },
+                    patch(url, data, httpConfig) {
+                        return __awaiter(this, void 0, void 0, function* () {
+                            try {
+                                const res = yield axiosCustom.patch(url, data, getHttpConfig(httpConfig));
+                                return res.data;
+                            }
+                            catch (e) {
+                                if (isExpiredCause(e)) {
+                                    yield renewToken();
+                                    const res = yield axiosCustom.patch(url, data, getHttpConfig(httpConfig));
+                                    return res.data;
+                                }
+                                throw e;
+                            }
+                        });
+                    },
+                    post(url, data, httpConfig) {
+                        return __awaiter(this, void 0, void 0, function* () {
+                            try {
+                                const res = yield axiosCustom.post(url, data, getHttpConfig(httpConfig));
+                                return res.data;
+                            }
+                            catch (e) {
+                                if (isExpiredCause(e)) {
+                                    yield renewToken();
+                                    const res = yield axiosCustom.post(url, data, getHttpConfig(httpConfig));
+                                    return res.data;
+                                }
+                                throw e;
+                            }
+                        });
+                    },
+                    put(url, data, httpConfig) {
+                        return __awaiter(this, void 0, void 0, function* () {
+                            try {
+                                const res = yield axiosCustom.put(url, data, getHttpConfig(httpConfig));
+                                return res.data;
+                            }
+                            catch (e) {
+                                if (isExpiredCause(e)) {
+                                    yield renewToken();
+                                    const res = yield axiosCustom.put(url, data, getHttpConfig(httpConfig));
+                                    return res.data;
+                                }
+                                throw e;
+                            }
+                        });
+                    },
+                },
+            }),
+            token,
+        };
+    });
+}
+exports.prepareCrowdinClient = prepareCrowdinClient;
+function refreshCrowdinCreds({ config, credentials }) {
+    return __awaiter(this, void 0, void 0, function* () {
         (0, _1.log)('Crowdin credentials have expired. Requesting a new credentials', config.logger);
         const newCredentials = yield refreshToken(config, credentials);
         (0, _1.log)('Saving updated crowdin credentials in the database', config.logger);
-        yield (0, storage_1.getStorage)().updateCrowdinCredentials({
+        const newCrowdinCredentials = {
             id: credentials.id,
             appSecret: credentials.appSecret,
             domain: credentials.domain,
@@ -78,15 +220,15 @@ function prepareCrowdinClient(config, credentials) {
             accessToken: newCredentials.accessToken,
             expire: (new Date().getTime() / 1000 + newCredentials.expiresIn).toString(),
             type: credentials.type,
-        });
+        };
+        yield (0, storage_1.getStorage)().updateCrowdinCredentials(newCrowdinCredentials);
         const token = (0, _1.decryptData)(config, newCredentials.accessToken);
         return {
-            client: new crowdin_api_client_1.default({ token, organization, baseUrl: (_b = config.crowdinUrls) === null || _b === void 0 ? void 0 : _b.apiUrl }),
+            credentials: newCrowdinCredentials,
             token,
         };
     });
 }
-exports.prepareCrowdinClient = prepareCrowdinClient;
 function prepareIntegrationCredentials(config, integration, integrationCredentials) {
     var _a, _b, _c, _d, _e, _f, _g, _h;
     return __awaiter(this, void 0, void 0, function* () {

package/out/util/cron.d.ts

@@ -1,5 +1,5 @@
 import Crowdin from '@crowdin/crowdin-api-client';
-import { Config, CronJob, IntegrationLogic, UpdateIntegrationRequest, Provider, IntegrationRequest } from '../models';
+import { Config, CronJob, IntegrationLogic, IntegrationRequest, Provider, UpdateIntegrationRequest } from '../models';
 export declare function runJob(config: Config, integration: IntegrationLogic, job: CronJob): Promise<void>;
 export declare function filesCron(config: Config, integration: IntegrationLogic, period: string): Promise<void>;
 export declare function skipFoldersFromIntegrationRequest(config: Config, integration: IntegrationLogic, projectId: number, crowdinFiles: UpdateIntegrationRequest, crowdinClient: Crowdin): Promise<UpdateIntegrationRequest>;

package/out/util/cron.js

@@ -45,16 +45,14 @@ function runJob(config, integration, job) {
         (0, _1.log)(`Starting cron job with expression [${job.expression}]`, config.logger);
         const crowdinCredentialsList = yield (0, storage_1.getStorage)().getAllCrowdinCredentials();
         yield Promise.all(crowdinCredentialsList.map((crowdinCredentials) => __awaiter(this, void 0, void 0, function* () {
-            const { token } = yield (0, connection_1.prepareCrowdinClient)(config, crowdinCredentials);
+            const { token, client: crowdinClient } = yield (0, connection_1.prepareCrowdinClient)(config, crowdinCredentials, true);
             const { expired } = yield (0, connection_1.checkSubscription)(config, token, crowdinCredentials.id, crowdinCredentials.type);
             if (expired) {
                 (0, _1.log)(`Subscription expired. Skipping job [${job.expression}] for organization ${crowdinCredentials.id}`);
                 return;
             }
             const integrationCredentialsList = yield (0, storage_1.getStorage)().getAllIntegrationCredentials(crowdinCredentials.id);
-            //executing in non-parallel way to properly prepare crowdin client (avoid expriration issues)
             for (const integrationCredentials of integrationCredentialsList) {
-                const { client: crowdinClient } = yield (0, connection_1.prepareCrowdinClient)(config, crowdinCredentials);
                 const projectId = crowdinAppFunctions.getProjectId(integrationCredentials.id);
                 const apiCredentials = yield (0, connection_1.prepareIntegrationCredentials)(config, integration, integrationCredentials);
                 const rootFolder = yield (0, defaults_1.getRootFolder)(config, integration, crowdinClient, projectId);
@@ -88,7 +86,7 @@ function filesCron(config, integration, period) {
                 return;
             }
             const projectId = crowdinAppFunctions.getProjectId(integrationCredentials.id);
-            const { client: crowdinClient, token } = yield (0, connection_1.prepareCrowdinClient)(config, crowdinCredentials);
+            const { client: crowdinClient, token } = yield (0, connection_1.prepareCrowdinClient)(config, crowdinCredentials, true);
             const { expired } = yield (0, connection_1.checkSubscription)(config, token, crowdinCredentials.id, crowdinCredentials.type);
             if (expired) {
                 (0, _1.log)(`Subscription expired. Skipping job [${period}] for organization ${crowdinCredentials.id}`);
@@ -96,8 +94,6 @@ function filesCron(config, integration, period) {
             }
             const rootFolder = yield (0, defaults_1.getRootFolder)(config, integration, crowdinClient, projectId);
             const apiCredentials = yield (0, connection_1.prepareIntegrationCredentials)(config, integration, integrationCredentials);
-            //split invocation into chunks to avoid expiration issues
-            const chunkSize = 10;
             if ((_a = integration.syncNewElements) === null || _a === void 0 ? void 0 : _a[syncSettings.provider]) {
                 newFiles = yield (0, file_snapshot_1.getAllNewFiles)({
                     config,
@@ -126,15 +122,8 @@ function filesCron(config, integration, period) {
                         return;
                     }
                 }
-                const filesArr = Object.keys(filesToProcess);
-                for (let i = 0; i < filesArr.length; i += chunkSize) {
-                    const chunk = filesArr.slice(i, i + chunkSize);
-                    const filesChunk = {};
-                    chunk.forEach((f) => (filesChunk[f] = filesToProcess[f]));
-                    const apiCredentials = yield (0, connection_1.prepareIntegrationCredentials)(config, integration, integrationCredentials);
-                    const { client } = yield (0, connection_1.prepareCrowdinClient)(config, crowdinCredentials);
-                    yield integration.updateIntegration(projectId, client, apiCredentials, filesChunk, rootFolder, intConfig);
-                }
+                const apiCredentials = yield (0, connection_1.prepareIntegrationCredentials)(config, integration, integrationCredentials);
+                yield integration.updateIntegration(projectId, crowdinClient, apiCredentials, filesToProcess, rootFolder, intConfig);
                 if (newFiles) {
                     yield (0, storage_1.getStorage)().updateSyncSettings(JSON.stringify(Object.assign(Object.assign({}, files), newFiles)), syncSettings.integrationId, syncSettings.crowdinId, 'schedule', syncSettings.provider);
                     const currentFileSnapshot = yield (0, file_snapshot_1.getCrowdinSnapshot)(config, integration, crowdinClient, projectId, intConfig);
@@ -150,12 +139,8 @@ function filesCron(config, integration, period) {
                     node_type: file.nodeType || file.node_type }, (file.type ? { type: file.type } : {}))));
                 const intFiles = allIntFiles.filter((file) => 'type' in file);
                 (0, _1.log)(`Executing updateCrowdin task for files cron job with period [${period}] for project ${projectId}. Files ${intFiles.length}`, config.logger);
-                for (let i = 0; i < intFiles.length; i += chunkSize) {
-                    const chunk = intFiles.slice(i, i + chunkSize);
-                    const apiCredentials = yield (0, connection_1.prepareIntegrationCredentials)(config, integration, integrationCredentials);
-                    const { client } = yield (0, connection_1.prepareCrowdinClient)(config, crowdinCredentials);
-                    yield integration.updateCrowdin(projectId, client, apiCredentials, chunk, rootFolder, intConfig);
-                }
+                const apiCredentials = yield (0, connection_1.prepareIntegrationCredentials)(config, integration, integrationCredentials);
+                yield integration.updateCrowdin(projectId, crowdinClient, apiCredentials, intFiles, rootFolder, intConfig);
                 const newSyncSettingsFiels = allIntFiles.map((file) => (Object.assign(Object.assign({}, file), { schedule: true, sync: false })));
                 if (newFiles) {
                     yield (0, storage_1.getStorage)().updateSyncSettings(JSON.stringify(newSyncSettingsFiels), syncSettings.integrationId, syncSettings.crowdinId, 'schedule', syncSettings.provider);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@crowdin/app-project-module",
-  "version": "0.30.1",
+  "version": "0.30.3",
   "description": "Module that generates for you all common endpoints for serving standalone Crowdin App",
   "main": "out/index.js",
   "types": "out/index.d.ts",
@@ -32,9 +32,9 @@
   "devDependencies": {
     "@babel/preset-react": "^7.22.3",
     "@emotion/react": "^11.11.1",
-    "@emotion/styled": "^11.10.6",
+    "@emotion/styled": "^11.11.0",
     "@mui/icons-material": "^5.11.16",
-    "@mui/material": "^5.11.12",
+    "@mui/material": "^5.14.2",
     "@rjsf/core": "^5.7.3",
     "@rjsf/mui": "^5.7.3",
     "@rjsf/utils": "^5.7.3",
@@ -44,14 +44,14 @@
     "@rollup/plugin-json": "^6.0.0",
     "@rollup/plugin-node-resolve": "^15.0.1",
     "@rollup/plugin-replace": "^5.0.2",
-    "@rollup/plugin-terser": "^0.4.0",
+    "@rollup/plugin-terser": "^0.4.3",
     "@types/amqplib": "^0.10.1",
     "@types/crypto-js": "^4.1.1",
     "@types/express": "4.17.17",
     "@types/express-handlebars": "^5.3.1",
     "@types/jest": "^29.5.2",
     "@types/node": "^12.20.55",
-    "@types/node-cron": "^3.0.7",
+    "@types/node-cron": "^3.0.8",
     "@types/swagger-jsdoc": "^6.0.1",
     "@typescript-eslint/eslint-plugin": "^2.3.1",
     "@typescript-eslint/parser": "^2.3.1",