@crowdin/app-project-module 0.39.0 → 0.40.0

package/out/util/files.d.ts

@@ -1,6 +1,7 @@
 /// <reference types="node" />
-import { ProcessFileString } from '../models';
+import { ProcessFileString, SkipIntegrationNodes, TreeItem } from '../models';
 export declare const MAX_BODY_SIZE: number;
 export declare function storeFile(fileContent: Buffer, folder: string): Promise<string>;
 export declare function getFileContent(url: string): Promise<Buffer>;
 export declare function getFileStrings(url: string): Promise<ProcessFileString[]>;
+export declare function skipFilesByRegex(files: TreeItem[] | undefined, skipIntegrationNodes?: SkipIntegrationNodes): TreeItem[];

package/out/util/files.js

@@ -12,7 +12,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getFileStrings = exports.getFileContent = exports.storeFile = exports.MAX_BODY_SIZE = void 0;
+exports.skipFilesByRegex = exports.getFileStrings = exports.getFileContent = exports.storeFile = exports.MAX_BODY_SIZE = void 0;
 const axios_1 = __importDefault(require("axios"));
 const fs_1 = __importDefault(require("fs"));
 const path_1 = __importDefault(require("path"));
@@ -45,3 +45,21 @@ function getFileStrings(url) {
     });
 }
 exports.getFileStrings = getFileStrings;
+function skipFilesByRegex(files, skipIntegrationNodes) {
+    if (!Array.isArray(files)) {
+        return [];
+    }
+    if (skipIntegrationNodes) {
+        files = files.filter((file) => file);
+        if (skipIntegrationNodes.fileNamePattern) {
+            const regex = new RegExp(skipIntegrationNodes.fileNamePattern);
+            files = files.filter((file) => !('type' in file) || !regex.test(file.name));
+        }
+        if (skipIntegrationNodes.folderNamePattern) {
+            const regex = new RegExp(skipIntegrationNodes.folderNamePattern);
+            files = files.filter((file) => 'type' in file || !regex.test(file.name));
+        }
+    }
+    return files;
+}
+exports.skipFilesByRegex = skipFilesByRegex;

package/out/util/index.js

@@ -56,7 +56,9 @@ function handleError(err, req, res) {
             res.redirect('/');
             return;
         }
-        res.status(code).send({ message: (0, logger_1.getErrorMessage)(err), code });
+        if (!res.headersSent) {
+            res.status(code).send({ message: (0, logger_1.getErrorMessage)(err), code });
+        }
     });
 }
 function runAsyncWrapper(callback) {

package/out/util/job.d.ts

@@ -0,0 +1,12 @@
+import { JobClient, JobType } from '../models/job';
+import { Response } from 'express';
+export declare function runAsJob({ integrationId, crowdinId, type, title, payload, res, jobCallback, onError, }: {
+    integrationId: string;
+    crowdinId: string;
+    type: JobType;
+    title?: string;
+    payload?: any;
+    res?: Response;
+    jobCallback: (arg1: JobClient) => Promise<any>;
+    onError?: (e: any) => Promise<void>;
+}): Promise<void>;

package/out/util/job.js

@@ -0,0 +1,88 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runAsJob = void 0;
+const job_1 = require("../models/job");
+const storage_1 = require("../storage");
+const logger_1 = require("./logger");
+const blockingJobs = {
+    [job_1.JobType.UPDATE_TO_CROWDIN]: [job_1.JobType.UPDATE_TO_CROWDIN, job_1.JobType.UPDATE_TO_INTEGRATION],
+    [job_1.JobType.UPDATE_TO_INTEGRATION]: [job_1.JobType.UPDATE_TO_CROWDIN, job_1.JobType.UPDATE_TO_INTEGRATION],
+};
+function runAsJob({ integrationId, crowdinId, type, title, payload, res, jobCallback, onError, }) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const storage = (0, storage_1.getStorage)();
+        const activeJobs = yield storage.getActiveJobs({ integrationId, crowdinId });
+        if (activeJobs === null || activeJobs === void 0 ? void 0 : activeJobs.length) {
+            const existingJob = activeJobs.find((job) => blockingJobs[type].includes(job.type));
+            if (existingJob === null || existingJob === void 0 ? void 0 : existingJob.id) {
+                if (res) {
+                    res.status(202).send({ jobId: existingJob.id, message: 'Job is already running' });
+                }
+                return;
+            }
+        }
+        const jobId = yield storage.createJob({
+            integrationId,
+            crowdinId,
+            type,
+            title: title || '',
+            payload: JSON.stringify(payload),
+        });
+        if (res) {
+            res.status(202).send({ jobId });
+        }
+        const job = {
+            get: function getJob() {
+                return __awaiter(this, void 0, void 0, function* () {
+                    return yield storage.getJob({ id: jobId });
+                });
+            },
+            update: function updateProgress({ progress, status, info, data }) {
+                return __awaiter(this, void 0, void 0, function* () {
+                    const prevData = yield this.get();
+                    if ((prevData === null || prevData === void 0 ? void 0 : prevData.status) === job_1.JobStatus.CANCELED) {
+                        return { isCanceled: true };
+                    }
+                    yield storage.updateJob({
+                        id: jobId,
+                        progress,
+                        status: status || job_1.JobStatus.IN_PROGRESS,
+                        info,
+                        data: JSON.stringify(data),
+                    });
+                    return { isCanceled: false };
+                });
+            },
+        };
+        try {
+            const data = yield jobCallback(job);
+            yield job.update({
+                progress: 100,
+                status: job_1.JobStatus.FINISHED,
+                data,
+            });
+        }
+        catch (e) {
+            yield job.update({
+                status: job_1.JobStatus.FAILED,
+                info: (0, logger_1.getErrorMessage)(e),
+            });
+            if (onError) {
+                yield onError(e);
+            }
+            else {
+                throw e;
+            }
+        }
+    });
+}
+exports.runAsJob = runAsJob;

package/out/util/logger.js

@@ -67,7 +67,9 @@ function log(message, context) {
         else {
             let prefix = `[${new Date().toISOString()}]`;
             if (context) {
+                logsFormatter.resetContext();
                 logsFormatter.setContext({
+                    appIdentifier: context.appIdentifier || '',
                     project: {
                         id: context.jwtPayload.context.project_id,
                         identifier: (_a = context.jwtPayload.context.project_identifier) !== null && _a !== void 0 ? _a : '',
@@ -100,7 +102,9 @@ function logError(e, context) {
     }
     else {
         if (context) {
+            logsFormatter.resetContext();
             logsFormatter.setContext({
+                appIdentifier: context.appIdentifier || '',
                 project: {
                     id: context.jwtPayload.context.project_id,
                     identifier: (_a = context.jwtPayload.context.project_identifier) !== null && _a !== void 0 ? _a : '',

package/out/util/webhooks.js

@@ -44,6 +44,8 @@ const connection_1 = require("./connection");
 const defaults_1 = require("./defaults");
 const index_1 = require("./index");
 const logger_1 = require("./logger");
+const prefetchCount = 20;
+const forceProcessDelay = 5000;
 exports.HookEvents = {
     fileAdded: 'file.added',
     fileDeleted: 'file.deleted',
@@ -308,7 +310,7 @@ function listenQueueMessage(config, integration, queueUrl, queueName) {
             const channel = yield connection.createChannel();
             if (channel) {
                 yield channel.assertQueue(queueName, { durable: true });
-                yield channel.prefetch(1);
+                yield channel.prefetch(prefetchCount);
                 yield channel.consume(queueName, consumer(channel, config, integration), { noAck: false });
             }
         }
@@ -321,23 +323,67 @@ function listenQueueMessage(config, integration, queueUrl, queueName) {
 }
 exports.listenQueueMessage = listenQueueMessage;
 function consumer(channel, config, integration) {
+    let messagesCounter = 0;
+    let webhooksInfo = {};
+    let webhooksData = [];
+    let timeoutId;
+    const resetStateVariables = () => {
+        messagesCounter = 0;
+        webhooksInfo = {};
+        webhooksData = [];
+    };
     return function (msg) {
         var _a;
         return __awaiter(this, void 0, void 0, function* () {
+            messagesCounter++;
             if (!msg) {
                 return;
             }
+            clearTimeout(timeoutId);
             try {
                 const data = JSON.parse(msg.content.toString());
                 const urlParam = (_a = integration.webhooks) === null || _a === void 0 ? void 0 : _a.urlParam;
                 const webhookUrlParam = data.query[urlParam];
-                const webhookData = yield prepareWebhookData(config, integration, webhookUrlParam, models_1.Provider.INTEGRATION);
-                yield updateCrowdinFromWebhookRequest({ integration, webhookData, req: data });
+                const { clientId } = decodedUrlParam(config, webhookUrlParam);
+                if (!webhooksInfo[clientId]) {
+                    webhooksInfo[clientId] = {};
+                    webhooksInfo[clientId].data = [data];
+                    webhooksInfo[clientId].integration = integration;
+                    webhooksData.push(prepareWebhookData(config, integration, webhookUrlParam, models_1.Provider.INTEGRATION).then((res) => {
+                        webhooksInfo[clientId].webhookData = res;
+                    }));
+                }
+                else {
+                    webhooksInfo[clientId].data.push(data);
+                }
+                if (messagesCounter < prefetchCount) {
+                    // if all messages are not received, wait 5 seconds to force process messages
+                    timeoutId = setTimeout(() => __awaiter(this, void 0, void 0, function* () {
+                        yield processMessages(webhooksData, webhooksInfo, channel, msg);
+                        resetStateVariables();
+                    }), forceProcessDelay);
+                    return;
+                }
+                clearTimeout(timeoutId);
+                yield processMessages(webhooksData, webhooksInfo, channel, msg);
+                resetStateVariables();
             }
             catch (e) {
                 (0, logger_1.logError)(e);
             }
-            channel.ack(msg);
         });
     };
 }
+function processMessages(webhooksData, webhooksInfo, channel, msg) {
+    return __awaiter(this, void 0, void 0, function* () {
+        yield Promise.all(webhooksData);
+        for (const { data, integration, webhookData } of Object.values(webhooksInfo)) {
+            yield updateCrowdinFromWebhookRequest({
+                integration: integration,
+                webhookData: webhookData,
+                req: data,
+            });
+        }
+        channel.ack(msg, true);
+    });
+}

package/out/views/main.handlebars

@@ -58,6 +58,7 @@
                 {{/if}}
         </div>
         <crowdin-simple-integration
+                async-progress
                 {{#if syncNewElements.crowdin}}
                     skip-crowdin-auto-schedule
                 {{/if}}
@@ -90,7 +91,6 @@
         >
         </crowdin-simple-integration>
         <div id="user-errors" class="hidden">
-            <crowdin-alert title="Error Logs">This table displays the most recent error logs from the past month. Logs older than one month will be automatically deleted.</crowdin-alert>
             <crowdin-show-as-table
                     is-loading
                     id="user-errors-table"
@@ -98,9 +98,13 @@
                     total-records="25"
                     search-placeholder="Search something"
             ></crowdin-show-as-table>
+            <crowdin-alert>This table displays the most recent error logs from the past month. Logs older than one month will be automatically deleted.</crowdin-alert>
         </div>
     </div>
     <crowdin-toasts></crowdin-toasts>
+    <crowdin-async-progress
+        cancelAsyncAction=""
+    ></crowdin-async-progress>
     <crowdin-modal id="subscription-modal" modal-width="50" close-button="false">
         <div>
             <crowdin-alert type="warning">Subscribe to continue using the {{name}} app.</crowdin-alert>
@@ -296,6 +300,9 @@
     });
     document.body.addEventListener('uploadFilesToCrowdin', uploadFilesToCrowdin);
     document.body.addEventListener('uploadFilesToIntegration', uploadFilesToIntegration);
+    document.body.addEventListener('cancelAsyncAction', (e) => {
+      cancelJob(e.detail);
+    });
     {{#if integrationSearchListener}}
         document.body.addEventListener("integrationFilterChange", (event) => {
             getIntegrationData(false, 0, event.detail);
@@ -308,11 +315,27 @@
     const fileType = '1';
     const branchType = '2';
 
+    const JOB_TYPE = {
+        updateCrowdin: 'updateCrowdin',
+        updateIntegration: 'updateIntegration',
+    };
+
+    const JOB_STATUS = {
+        created: 'created',
+        inProgress: 'inProgress',
+        failed: 'failed',
+        canceled: 'canceled',
+        finished: 'finished',
+    };
+
+    const asyncJobs = {};
+
     let project = {};
     let crowdinData = [];
 
     getCrowdinData();
     getIntegrationData();
+    getActiveJobs();
 
     function integrationLogout() {
         checkOrigin()
@@ -481,10 +504,131 @@
                 body: JSON.stringify(req)
             }))
             .then(checkResponse)
-            .then(() => showToast(`File${event.detail.length > 1 ? 's' : ''} imported successfully`))
-            .then(getCrowdinData)
+            .then((res) => {
+                checkJob({
+                    jobId: res?.jobId,
+                    jobType: JOB_TYPE.updateCrowdin,
+                })
+            })
             .catch(e => catchRejection(e, 'Can\'t upload templates to Crowdin'))
-            .finally(() => (appComponent.setAttribute('is-to-crowdin-process', false)));
+    }
+
+    function getActiveJobs() {
+        checkOrigin()
+            .then(restParams => fetch(`api/jobs${restParams}`))
+            .then(checkResponse)
+            .then((jobs) => {
+                if (!Array.isArray(jobs)) {
+                  return;
+                }
+
+                jobs.forEach((job) => {
+                    switch (job.type) {
+                        case JOB_TYPE.updateCrowdin:
+                            appComponent.setAttribute('is-to-crowdin-process', true);
+                            break;
+                        case JOB_TYPE.updateIntegration:
+                            appComponent.setAttribute('is-to-integration-process', true);
+                            break;
+                        default:
+                    }
+
+                    checkJob({
+                        jobId: job.id,
+                        jobType: job.type,
+                    })
+                })
+            })
+            .catch(e => catchRejection(e, 'Sync status check failed'))
+    }
+
+    function cancelJob(jobId) {
+        if (asyncJobs[jobId]?.isFailed) {
+            return;
+        }
+
+        checkOrigin()
+            .then(restParams => fetch('api/jobs' + restParams + '&job_id=' + jobId, {
+                method: 'DELETE',
+                headers: { 'Content-Type': 'application/json' },
+            }))
+            .then(checkResponse)
+            .then(() => showToast('Cancellation…'))
+            .catch(e => catchRejection(e, 'Sync cancellation failed'));
+    }
+
+    function checkJob({jobId, jobType, onSuccess, onError, onFinally}) {
+        switch (jobType) {
+            case JOB_TYPE.updateCrowdin:
+                if (!onSuccess) {
+                    onSuccess = ((job) => {
+                        getCrowdinData();
+                    });
+                }
+
+                if (!onFinally) {
+                    onFinally = (() => appComponent.setAttribute('is-to-crowdin-process', false));
+                }
+                break;
+            case JOB_TYPE.updateIntegration:
+                if (!onFinally) {
+                    onFinally = (() => appComponent.setAttribute('is-to-integration-process', false));
+                }
+                break;
+            default:
+        }
+
+        checkOrigin()
+            .then(restParams => fetch('api/jobs' + restParams + '&job_id=' + jobId))
+            .then(checkResponse)
+            .then((job) => {
+                const isFailed = [JOB_STATUS.failed, JOB_STATUS.canceled].includes(job.status);
+                const progress = isFailed || JOB_STATUS.finished === job.status ? 100 : job.progress;
+                const info = JOB_STATUS.canceled === job.status ? `Cancelled\n${job.info || ''}` : job.info;
+
+                pushJobs([ {
+                    id: job.id,
+                    title: job.title,
+                    progress,
+                    info,
+                    isFailed,
+                } ]);
+
+                if (isFailed) {
+                    onError && onError();
+                } else if ([JOB_STATUS.created, JOB_STATUS.inProgress].includes(job.status) && job.progress < 100) {
+                    setTimeout(() => {
+                        checkJob({jobId, jobType, onSuccess, onError, onFinally});
+                    }, {{asyncProgress.checkInterval}});
+                    return;
+                } else {
+                    onSuccess && onSuccess(job);
+                }
+
+                onFinally && onFinally(job);
+            })
+            .catch((e) => {
+                onFinally && onFinally();
+
+                pushJobs([ {
+                    id: jobId,
+                    isFailed: true,
+                } ]);
+
+                showToast('Sync status check failed');
+            });
+    }
+
+    function pushJobs(jobs) {
+        const el = document.querySelector('crowdin-async-progress');
+        if (el && el.pushJobs) {
+            el.pushJobs(jobs);
+            jobs.forEach((job) => (asyncJobs[job.id] = job));
+        } else {
+            setTimeout(() => {
+                pushJobs(jobs);
+            }, 300);
+        }
     }
 
     function uploadFilesToIntegration(e) {
@@ -504,9 +648,13 @@
                 body: JSON.stringify(req)
             }))
             .then(checkResponse)
-            .then(() => showToast(`File${Object.keys(e.detail).length > 1 ? 's' : ''} exported successfully`))
+            .then((res) => {
+                checkJob({
+                    jobId: res?.jobId,
+                    jobType: JOB_TYPE.updateIntegration,
+                })
+            })
             .catch(e => catchRejection(e, 'Can\'t upload files to {{name}}'))
-            .finally(() => (appComponent.setAttribute('is-to-integration-process', false)));
     }
 
     {{#if configurationFields}}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@crowdin/app-project-module",
-  "version": "0.39.0",
+  "version": "0.40.0",
   "description": "Module that generates for you all common endpoints for serving standalone Crowdin App",
   "main": "out/index.js",
   "types": "out/index.d.ts",
@@ -15,11 +15,10 @@
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.423.0",
-    "@aws-sdk/s3-request-presigner": "^3.423.0",
+    "@aws-sdk/s3-request-presigner": "^3.484.0",
     "@crowdin/crowdin-apps-functions": "0.6.0",
-    "@crowdin/logs-formatter": "^2.0.6",
+    "@crowdin/logs-formatter": "^2.1.1",
     "@godaddy/terminus": "^4.12.1",
-    "@types/pg": "^8.10.3",
     "amqplib": "^0.10.3",
     "crypto-js": "^4.2.0",
     "express": "4.18.2",
@@ -33,29 +32,31 @@
     "uuid": "^8.3.2"
   },
   "devDependencies": {
-    "@babel/preset-react": "^7.22.15",
+    "@babel/preset-react": "^7.23.3",
     "@emotion/react": "^11.11.1",
     "@emotion/styled": "^11.11.0",
-    "@mui/icons-material": "^5.14.16",
-    "@mui/material": "^5.14.16",
-    "@rjsf/core": "^5.13.3",
-    "@rjsf/mui": "^5.13.3",
-    "@rjsf/utils": "^5.13.3",
-    "@rjsf/validator-ajv8": "^5.13.3",
+    "@mui/icons-material": "^5.15.2",
+    "@mui/material": "^5.15.2",
+    "@rjsf/core": "^5.15.1",
+    "@rjsf/mui": "^5.15.1",
+    "@rjsf/utils": "^5.15.1",
+    "@rjsf/validator-ajv8": "^5.15.1",
     "@rollup/plugin-babel": "^6.0.4",
-    "@rollup/plugin-commonjs": "^24.0.1",
+    "@rollup/plugin-commonjs": "^24.1.0",
     "@rollup/plugin-json": "^6.0.0",
     "@rollup/plugin-node-resolve": "^15.2.1",
-    "@rollup/plugin-replace": "^5.0.2",
+    "@rollup/plugin-replace": "^5.0.5",
     "@rollup/plugin-terser": "^0.4.3",
-    "@types/amqplib": "^0.10.2",
+    "@types/amqplib": "^0.10.4",
     "@types/crypto-js": "^4.1.3",
     "@types/express": "4.17.18",
     "@types/express-handlebars": "^5.3.1",
     "@types/jest": "^29.5.5",
-    "@types/node": "^16.18.60",
+    "@types/node": "^16.18.69",
     "@types/node-cron": "^3.0.9",
+    "@types/pg": "^8.10.3",
     "@types/swagger-jsdoc": "^6.0.1",
+    "@types/uuid": "^9.0.7",
     "@typescript-eslint/eslint-plugin": "^2.3.1",
     "@typescript-eslint/parser": "^2.3.1",
     "eslint": "^6.4.0",

package/out/handlers/user-errors.d.ts

@@ -1,3 +0,0 @@
-/// <reference types="qs" />
-import { Response } from 'express';
-export default function handle(): (req: import("../models").CrowdinClientRequest | import("express").Request<import("express-serve-static-core").ParamsDictionary, any, any, import("qs").ParsedQs, Record<string, any>>, res: Response<any, Record<string, any>>, next: Function) => void;