@crowdin/app-project-module 0.39.1 → 0.41.0

package/out/util/webhooks.js

@@ -44,6 +44,8 @@ const connection_1 = require("./connection");
 const defaults_1 = require("./defaults");
 const index_1 = require("./index");
 const logger_1 = require("./logger");
+const prefetchCount = 20;
+const forceProcessDelay = 5000;
 exports.HookEvents = {
     fileAdded: 'file.added',
     fileDeleted: 'file.deleted',
@@ -227,6 +229,7 @@ function prepareWebhookData(config, integration, webhookUrlParam, provider) {
         const { projectId, crowdinId, clientId } = decodedUrlParam(config, webhookUrlParam);
         const crowdinCredentials = yield (0, storage_1.getStorage)().getCrowdinCredentials(crowdinId);
         const integrationCredentials = yield (0, storage_1.getStorage)().getIntegrationCredentials(clientId);
+        const integrationConfig = yield (0, storage_1.getStorage)().getIntegrationConfig(clientId);
         const context = {
             jwtPayload: {
                 context: {
@@ -245,8 +248,8 @@ function prepareWebhookData(config, integration, webhookUrlParam, provider) {
             context,
         });
         const preparedIntegrationCredentials = yield (0, connection_1.prepareIntegrationCredentials)(config, integration, integrationCredentials);
-        if (integrationCredentials === null || integrationCredentials === void 0 ? void 0 : integrationCredentials.config) {
-            appSettings = JSON.parse(integrationCredentials.config);
+        if (integrationConfig === null || integrationConfig === void 0 ? void 0 : integrationConfig.config) {
+            appSettings = JSON.parse(integrationConfig.config);
             const isWebhookSync = Number(appSettings.schedule) !== models_1.SyncSchedule.DISABLED;
             if (isWebhookSync) {
                 syncSettings = (yield (0, storage_1.getStorage)().getSyncSettings(clientId, crowdinId, 'schedule', provider));
@@ -308,7 +311,7 @@ function listenQueueMessage(config, integration, queueUrl, queueName) {
             const channel = yield connection.createChannel();
             if (channel) {
                 yield channel.assertQueue(queueName, { durable: true });
-                yield channel.prefetch(1);
+                yield channel.prefetch(prefetchCount);
                 yield channel.consume(queueName, consumer(channel, config, integration), { noAck: false });
             }
         }
@@ -321,23 +324,67 @@ function listenQueueMessage(config, integration, queueUrl, queueName) {
 }
 exports.listenQueueMessage = listenQueueMessage;
 function consumer(channel, config, integration) {
+    let messagesCounter = 0;
+    let webhooksInfo = {};
+    let webhooksData = [];
+    let timeoutId;
+    const resetStateVariables = () => {
+        messagesCounter = 0;
+        webhooksInfo = {};
+        webhooksData = [];
+    };
     return function (msg) {
         var _a;
         return __awaiter(this, void 0, void 0, function* () {
+            messagesCounter++;
             if (!msg) {
                 return;
             }
+            clearTimeout(timeoutId);
             try {
                 const data = JSON.parse(msg.content.toString());
                 const urlParam = (_a = integration.webhooks) === null || _a === void 0 ? void 0 : _a.urlParam;
                 const webhookUrlParam = data.query[urlParam];
-                const webhookData = yield prepareWebhookData(config, integration, webhookUrlParam, models_1.Provider.INTEGRATION);
-                yield updateCrowdinFromWebhookRequest({ integration, webhookData, req: data });
+                const { clientId } = decodedUrlParam(config, webhookUrlParam);
+                if (!webhooksInfo[clientId]) {
+                    webhooksInfo[clientId] = {};
+                    webhooksInfo[clientId].data = [data];
+                    webhooksInfo[clientId].integration = integration;
+                    webhooksData.push(prepareWebhookData(config, integration, webhookUrlParam, models_1.Provider.INTEGRATION).then((res) => {
+                        webhooksInfo[clientId].webhookData = res;
+                    }));
+                }
+                else {
+                    webhooksInfo[clientId].data.push(data);
+                }
+                if (messagesCounter < prefetchCount) {
+                    // if all messages are not received, wait 5 seconds to force process messages
+                    timeoutId = setTimeout(() => __awaiter(this, void 0, void 0, function* () {
+                        yield processMessages(webhooksData, webhooksInfo, channel, msg);
+                        resetStateVariables();
+                    }), forceProcessDelay);
+                    return;
+                }
+                clearTimeout(timeoutId);
+                yield processMessages(webhooksData, webhooksInfo, channel, msg);
+                resetStateVariables();
             }
             catch (e) {
                 (0, logger_1.logError)(e);
             }
-            channel.ack(msg);
         });
     };
 }
+function processMessages(webhooksData, webhooksInfo, channel, msg) {
+    return __awaiter(this, void 0, void 0, function* () {
+        yield Promise.all(webhooksData);
+        for (const { data, integration, webhookData } of Object.values(webhooksInfo)) {
+            yield updateCrowdinFromWebhookRequest({
+                integration: integration,
+                webhookData: webhookData,
+                req: data,
+            });
+        }
+        channel.ack(msg, true);
+    });
+}

package/out/views/main.handlebars

@@ -58,6 +58,7 @@
                 {{/if}}
         </div>
         <crowdin-simple-integration
+                async-progress
                 {{#if syncNewElements.crowdin}}
                     skip-crowdin-auto-schedule
                 {{/if}}
@@ -101,6 +102,9 @@
         </div>
     </div>
     <crowdin-toasts></crowdin-toasts>
+    <crowdin-async-progress
+        cancelAsyncAction=""
+    ></crowdin-async-progress>
     <crowdin-modal id="subscription-modal" modal-width="50" close-button="false">
         <div>
             <crowdin-alert type="warning">Subscribe to continue using the {{name}} app.</crowdin-alert>
@@ -296,6 +300,9 @@
     });
     document.body.addEventListener('uploadFilesToCrowdin', uploadFilesToCrowdin);
     document.body.addEventListener('uploadFilesToIntegration', uploadFilesToIntegration);
+    document.body.addEventListener('cancelAsyncAction', (e) => {
+      cancelJob(e.detail);
+    });
     {{#if integrationSearchListener}}
         document.body.addEventListener("integrationFilterChange", (event) => {
             getIntegrationData(false, 0, event.detail);
@@ -308,11 +315,27 @@
     const fileType = '1';
     const branchType = '2';
 
+    const JOB_TYPE = {
+        updateCrowdin: 'updateCrowdin',
+        updateIntegration: 'updateIntegration',
+    };
+
+    const JOB_STATUS = {
+        created: 'created',
+        inProgress: 'inProgress',
+        failed: 'failed',
+        canceled: 'canceled',
+        finished: 'finished',
+    };
+
+    const asyncJobs = {};
+
     let project = {};
     let crowdinData = [];
 
     getCrowdinData();
     getIntegrationData();
+    getActiveJobs();
 
     function integrationLogout() {
         checkOrigin()
@@ -481,10 +504,131 @@
                 body: JSON.stringify(req)
             }))
             .then(checkResponse)
-            .then(() => showToast(`File${event.detail.length > 1 ? 's' : ''} imported successfully`))
-            .then(getCrowdinData)
+            .then((res) => {
+                checkJob({
+                    jobId: res?.jobId,
+                    jobType: JOB_TYPE.updateCrowdin,
+                })
+            })
             .catch(e => catchRejection(e, 'Can\'t upload templates to Crowdin'))
-            .finally(() => (appComponent.setAttribute('is-to-crowdin-process', false)));
+    }
+
+    function getActiveJobs() {
+        checkOrigin()
+            .then(restParams => fetch(`api/jobs${restParams}`))
+            .then(checkResponse)
+            .then((jobs) => {
+                if (!Array.isArray(jobs)) {
+                  return;
+                }
+
+                jobs.forEach((job) => {
+                    switch (job.type) {
+                        case JOB_TYPE.updateCrowdin:
+                            appComponent.setAttribute('is-to-crowdin-process', true);
+                            break;
+                        case JOB_TYPE.updateIntegration:
+                            appComponent.setAttribute('is-to-integration-process', true);
+                            break;
+                        default:
+                    }
+
+                    checkJob({
+                        jobId: job.id,
+                        jobType: job.type,
+                    })
+                })
+            })
+            .catch(e => catchRejection(e, 'Sync status check failed'))
+    }
+
+    function cancelJob(jobId) {
+        if (asyncJobs[jobId]?.isFailed) {
+            return;
+        }
+
+        checkOrigin()
+            .then(restParams => fetch('api/jobs' + restParams + '&job_id=' + jobId, {
+                method: 'DELETE',
+                headers: { 'Content-Type': 'application/json' },
+            }))
+            .then(checkResponse)
+            .then(() => showToast('Cancellation…'))
+            .catch(e => catchRejection(e, 'Sync cancellation failed'));
+    }
+
+    function checkJob({jobId, jobType, onSuccess, onError, onFinally}) {
+        switch (jobType) {
+            case JOB_TYPE.updateCrowdin:
+                if (!onSuccess) {
+                    onSuccess = ((job) => {
+                        getCrowdinData();
+                    });
+                }
+
+                if (!onFinally) {
+                    onFinally = (() => appComponent.setAttribute('is-to-crowdin-process', false));
+                }
+                break;
+            case JOB_TYPE.updateIntegration:
+                if (!onFinally) {
+                    onFinally = (() => appComponent.setAttribute('is-to-integration-process', false));
+                }
+                break;
+            default:
+        }
+
+        checkOrigin()
+            .then(restParams => fetch('api/jobs' + restParams + '&job_id=' + jobId))
+            .then(checkResponse)
+            .then((job) => {
+                const isFailed = [JOB_STATUS.failed, JOB_STATUS.canceled].includes(job.status);
+                const progress = isFailed || JOB_STATUS.finished === job.status ? 100 : job.progress;
+                const info = JOB_STATUS.canceled === job.status ? `Cancelled\n${job.info || ''}` : job.info;
+
+                pushJobs([ {
+                    id: job.id,
+                    title: job.title,
+                    progress,
+                    info,
+                    isFailed,
+                } ]);
+
+                if (isFailed) {
+                    onError && onError();
+                } else if ([JOB_STATUS.created, JOB_STATUS.inProgress].includes(job.status) && job.progress < 100) {
+                    setTimeout(() => {
+                        checkJob({jobId, jobType, onSuccess, onError, onFinally});
+                    }, {{asyncProgress.checkInterval}});
+                    return;
+                } else {
+                    onSuccess && onSuccess(job);
+                }
+
+                onFinally && onFinally(job);
+            })
+            .catch((e) => {
+                onFinally && onFinally();
+
+                pushJobs([ {
+                    id: jobId,
+                    isFailed: true,
+                } ]);
+
+                showToast('Sync status check failed');
+            });
+    }
+
+    function pushJobs(jobs) {
+        const el = document.querySelector('crowdin-async-progress');
+        if (el && el.pushJobs) {
+            el.pushJobs(jobs);
+            jobs.forEach((job) => (asyncJobs[job.id] = job));
+        } else {
+            setTimeout(() => {
+                pushJobs(jobs);
+            }, 300);
+        }
     }
 
     function uploadFilesToIntegration(e) {
@@ -504,9 +648,13 @@
                 body: JSON.stringify(req)
             }))
             .then(checkResponse)
-            .then(() => showToast(`File${Object.keys(e.detail).length > 1 ? 's' : ''} exported successfully`))
+            .then((res) => {
+                checkJob({
+                    jobId: res?.jobId,
+                    jobType: JOB_TYPE.updateIntegration,
+                })
+            })
             .catch(e => catchRejection(e, 'Can\'t upload files to {{name}}'))
-            .finally(() => (appComponent.setAttribute('is-to-integration-process', false)));
     }
 
     {{#if configurationFields}}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@crowdin/app-project-module",
-  "version": "0.39.1",
+  "version": "0.41.0",
   "description": "Module that generates for you all common endpoints for serving standalone Crowdin App",
   "main": "out/index.js",
   "types": "out/index.d.ts",
@@ -15,11 +15,10 @@
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.423.0",
-    "@aws-sdk/s3-request-presigner": "^3.423.0",
+    "@aws-sdk/s3-request-presigner": "^3.484.0",
     "@crowdin/crowdin-apps-functions": "0.6.0",
-    "@crowdin/logs-formatter": "^2.0.6",
+    "@crowdin/logs-formatter": "^2.1.2",
     "@godaddy/terminus": "^4.12.1",
-    "@types/pg": "^8.10.3",
     "amqplib": "^0.10.3",
     "crypto-js": "^4.2.0",
     "express": "4.18.2",
@@ -33,29 +32,31 @@
     "uuid": "^8.3.2"
   },
   "devDependencies": {
-    "@babel/preset-react": "^7.22.15",
+    "@babel/preset-react": "^7.23.3",
     "@emotion/react": "^11.11.1",
     "@emotion/styled": "^11.11.0",
-    "@mui/icons-material": "^5.14.19",
-    "@mui/material": "^5.14.19",
-    "@rjsf/core": "^5.15.0",
-    "@rjsf/mui": "^5.15.0",
-    "@rjsf/utils": "^5.15.0",
-    "@rjsf/validator-ajv8": "^5.15.0",
+    "@mui/icons-material": "^5.15.7",
+    "@mui/material": "^5.15.7",
+    "@rjsf/core": "^5.16.1",
+    "@rjsf/mui": "^5.16.1",
+    "@rjsf/utils": "^5.15.1",
+    "@rjsf/validator-ajv8": "^5.16.1",
     "@rollup/plugin-babel": "^6.0.4",
     "@rollup/plugin-commonjs": "^24.1.0",
     "@rollup/plugin-json": "^6.0.0",
-    "@rollup/plugin-node-resolve": "^15.2.1",
+    "@rollup/plugin-node-resolve": "^15.2.3",
     "@rollup/plugin-replace": "^5.0.5",
     "@rollup/plugin-terser": "^0.4.3",
     "@types/amqplib": "^0.10.4",
     "@types/crypto-js": "^4.1.3",
-    "@types/express": "4.17.18",
+    "@types/express": "4.17.21",
     "@types/express-handlebars": "^5.3.1",
     "@types/jest": "^29.5.5",
-    "@types/node": "^16.18.60",
+    "@types/node": "^16.18.69",
     "@types/node-cron": "^3.0.9",
-    "@types/swagger-jsdoc": "^6.0.1",
+    "@types/pg": "^8.10.3",
+    "@types/swagger-jsdoc": "^6.0.4",
+    "@types/uuid": "^9.0.7",
     "@typescript-eslint/eslint-plugin": "^2.3.1",
     "@typescript-eslint/parser": "^2.3.1",
     "eslint": "^6.4.0",

package/out/handlers/user-errors.d.ts

@@ -1,3 +0,0 @@
-/// <reference types="qs" />
-import { Response } from 'express';
-export default function handle(): (req: import("../models").CrowdinClientRequest | import("express").Request<import("express-serve-static-core").ParamsDictionary, any, any, import("qs").ParsedQs, Record<string, any>>, res: Response<any, Record<string, any>>, next: Function) => void;