@crowdin/app-project-module 0.28.11 → 0.29.0

package/out/util/file-snapshot.js

@@ -0,0 +1,144 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createOrUpdateFileSnapshot = exports.getIntegrationSnapshot = exports.getCrowdinSnapshot = exports.getAllNewFiles = exports.getFileDiff = void 0;
+const models_1 = require("../models");
+const index_1 = require("./index");
+const defaults_1 = require("./defaults");
+const storage_1 = require("../storage");
+function getFileDiff(currentFiles, savedFiles) {
+    return currentFiles.filter((x) => !savedFiles.some((x2) => x2.id === x.id));
+}
+exports.getFileDiff = getFileDiff;
+function getAllNewFiles(args) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const { config, integration, crowdinApiClient, crowdinId, integrationCredentials, integrationId, projectId, integrationSettings, syncSettings, } = args;
+        let currentFileSnapshot = [];
+        const fileSnapshotData = yield (0, storage_1.getStorage)().getFilesSnapshot(integrationId, crowdinId, syncSettings.provider);
+        const snapshotFiles = (fileSnapshotData === null || fileSnapshotData === void 0 ? void 0 : fileSnapshotData.files) ? JSON.parse(fileSnapshotData.files) : [];
+        if (syncSettings.provider === models_1.Provider.CROWDIN) {
+            currentFileSnapshot = yield getCrowdinSnapshot(config, integration, crowdinApiClient, projectId, integrationSettings);
+        }
+        else {
+            currentFileSnapshot = yield getIntegrationSnapshot(integration, integrationCredentials, integrationSettings);
+        }
+        const difference = getFileDiff(currentFileSnapshot, snapshotFiles);
+        const onlyFiles = difference.filter((file) => 'type' in file);
+        const synFiles = JSON.parse(syncSettings.files);
+        if (syncSettings.provider === models_1.Provider.INTEGRATION) {
+            if (integrationSettings[`new-${syncSettings.provider}-files`]) {
+                return onlyFiles;
+            }
+            const syncFolders = synFiles.filter((file) => !('type' in file));
+            return getNewFoldersFile(syncFolders, difference);
+        }
+        else {
+            const files = {};
+            const projectData = yield crowdinApiClient.projectsGroupsApi.getProject(projectId);
+            const targetLanguages = projectData.data.targetLanguageIds;
+            if (integrationSettings[`new-${syncSettings.provider}-files`]) {
+                for (const file of onlyFiles) {
+                    files[file.id] = targetLanguages;
+                }
+            }
+            else {
+                const syncFolders = currentFileSnapshot.filter((file) => !('type' in file) && Object.keys(synFiles).includes(file.id));
+                const newFiles = getNewFoldersFile(syncFolders, difference);
+                for (const file of newFiles) {
+                    files[file.id] = targetLanguages;
+                }
+            }
+            return files;
+        }
+    });
+}
+exports.getAllNewFiles = getAllNewFiles;
+function getNewFoldersFile(folders, snapshotFiles) {
+    let files = [];
+    for (const folder of folders) {
+        const newFiles = snapshotFiles.find((file) => file.parentId === folder.id);
+        if (newFiles) {
+            files = files.concat(newFiles);
+        }
+    }
+    files = files.filter((file) => 'type' in file);
+    return files;
+}
+function getCrowdinSnapshot(config, integration, crowdinApiClient, projectId, integrationSettings) {
+    return __awaiter(this, void 0, void 0, function* () {
+        let files = [];
+        if (integration.getCrowdinFiles) {
+            const rootFolder = yield (0, defaults_1.getRootFolder)(config, integration, crowdinApiClient, projectId);
+            files = yield integration.getCrowdinFiles(projectId, crowdinApiClient, rootFolder, integrationSettings);
+        }
+        return files;
+    });
+}
+exports.getCrowdinSnapshot = getCrowdinSnapshot;
+function getTreeItems(integrationData) {
+    let files = [];
+    if ((0, index_1.isExtendedResultType)(integrationData)) {
+        files = integrationData.data;
+    }
+    else {
+        files = integrationData;
+    }
+    return files;
+}
+function getOneLevelFetchingFiles(integration, integrationCredentials, integrationSettings, parentFiles) {
+    return __awaiter(this, void 0, void 0, function* () {
+        for (const file of parentFiles) {
+            if (!('type' in file)) {
+                let childs = yield integration.getIntegrationFiles(integrationCredentials, integrationSettings, file.id, '', 0);
+                childs = getTreeItems(childs);
+                if (childs.length > 0) {
+                    const childFiles = yield getOneLevelFetchingFiles(integration, integrationCredentials, integrationSettings, childs);
+                    parentFiles = [...parentFiles, ...childFiles];
+                }
+            }
+        }
+        return parentFiles;
+    });
+}
+function getIntegrationSnapshot(integration, integrationCredentials, integrationSettings) {
+    return __awaiter(this, void 0, void 0, function* () {
+        let files = [];
+        let integrationData = [];
+        integrationData = yield integration.getIntegrationFiles(integrationCredentials, integrationSettings, '', '', 0);
+        files = getTreeItems(integrationData);
+        if (integration.integrationOneLevelFetching) {
+            files = yield getOneLevelFetchingFiles(integration, integrationCredentials, integrationSettings, files);
+        }
+        return files;
+    });
+}
+exports.getIntegrationSnapshot = getIntegrationSnapshot;
+function createOrUpdateFileSnapshot(config, integration, req, provider) {
+    return __awaiter(this, void 0, void 0, function* () {
+        let files = [];
+        const existingSnapshot = yield (0, storage_1.getStorage)().getFilesSnapshot(req.crowdinContext.clientId, req.crowdinContext.crowdinId, provider);
+        if (provider === models_1.Provider.CROWDIN) {
+            files = yield getCrowdinSnapshot(config, integration, req.crowdinApiClient, req.crowdinContext.jwtPayload.context.project_id, req.integrationSettings);
+        }
+        if (provider === models_1.Provider.INTEGRATION) {
+            files = yield getIntegrationSnapshot(integration, req.integrationCredentials, req.integrationSettings);
+        }
+        if (existingSnapshot) {
+            (0, index_1.log)(`Updating file snapshot for provider ${provider} ${JSON.stringify(files, null, 2)}`, config.logger);
+            yield (0, storage_1.getStorage)().updateFilesSnapshot(JSON.stringify(files), req.crowdinContext.clientId, req.crowdinContext.crowdinId, provider);
+        }
+        else {
+            (0, index_1.log)(`Saving file snapshot for provider ${provider} ${JSON.stringify(files, null, 2)}`, config.logger);
+            yield (0, storage_1.getStorage)().saveFilesSnapshot(JSON.stringify(files), req.crowdinContext.clientId, req.crowdinContext.crowdinId, provider);
+        }
+    });
+}
+exports.createOrUpdateFileSnapshot = createOrUpdateFileSnapshot;

package/out/util/webhooks.d.ts

@@ -1,5 +1,5 @@
 import Crowdin from '@crowdin/crowdin-api-client';
-import { Config, CrowdinContextInfo, IntegrationLogic, Payload, UpdateIntegrationRequest, WebhookUrlParams } from '../models';
+import { Config, CrowdinContextInfo, IntegrationLogic, IntegrationSyncSettings, Payload, Provider, TreeItem, UpdateIntegrationRequest, WebhookUrlParams } from '../models';
 import { WebhooksModel } from '@crowdin/crowdin-api-client/out/webhooks';
 import { Request } from 'express';
 export declare function encodedUrlParam(config: Config, integration: IntegrationLogic, crowdinContext: CrowdinContextInfo): string;
@@ -14,7 +14,7 @@ export declare function updateCrowdinWebhooks(config: Config, client: Crowdin, p
 export declare function unregisterCrowdinWebhooks(config: Config, client: Crowdin, projectId: number, webhook: WebhooksModel.Webhook): Promise<void>;
 export declare function unregisterAllCrowdinWebhooks(config: Config, integration: IntegrationLogic, crowdinId: string): Promise<void>;
 export declare function filterSyncFiles(events: Payload[], syncFileSettings: UpdateIntegrationRequest): UpdateIntegrationRequest;
-export declare function prepareWebhookData(config: Config, integration: IntegrationLogic, webhookUrlParam: string, provider: string): Promise<{
+export declare function prepareWebhookData(config: Config, integration: IntegrationLogic, webhookUrlParam: string, provider: Provider): Promise<{
     projectId: number;
     crowdinClient: {
         client: Crowdin;
@@ -23,7 +23,8 @@ export declare function prepareWebhookData(config: Config, integration: Integrat
     preparedIntegrationCredentials: any;
     rootFolder: import("@crowdin/crowdin-api-client").SourceFilesModel.Directory | undefined;
     appSettings: any;
-    syncSettings: import("../models").IntegrationSyncSettings | null | undefined;
+    syncSettings: IntegrationSyncSettings | null;
+    newFiles: TreeItem[] | UpdateIntegrationRequest;
 }>;
 export declare function updateCrowdinFromWebhookRequest(integration: IntegrationLogic, webhookData: any, req: Request): Promise<void | import("../models").ExtendedResult<void>>;
 export declare function listenQueueMessage(config: Config, integration: IntegrationLogic, queueUrl: string, queueName: string): Promise<void>;

package/out/util/webhooks.js

@@ -44,6 +44,7 @@ const connection_1 = require("./connection");
 const storage_1 = require("../storage");
 const defaults_1 = require("./defaults");
 const util_1 = require("../util");
+const file_snapshot_1 = require("./file-snapshot");
 const HookEvents = {
     ALL: ['file.translated', 'file.approved'],
     TRANSLATED: ['file.translated'],
@@ -227,10 +228,12 @@ function filterSyncFiles(events, syncFileSettings) {
 }
 exports.filterSyncFiles = filterSyncFiles;
 function prepareWebhookData(config, integration, webhookUrlParam, provider) {
+    var _a;
     return __awaiter(this, void 0, void 0, function* () {
         let rootFolder = undefined;
         let appSettings = null;
         let syncSettings = null;
+        let newFiles = [];
         const { projectId, crowdinId, clientId } = decodedUrlParam(config, webhookUrlParam);
         const crowdinCredentials = yield (0, storage_1.getStorage)().getCrowdinCredentials(crowdinId);
         const crowdinClient = yield (0, connection_1.prepareCrowdinClient)(config, crowdinCredentials);
@@ -240,11 +243,32 @@ function prepareWebhookData(config, integration, webhookUrlParam, provider) {
             appSettings = JSON.parse(integrationCredentials.config);
             const isWebhookSync = +appSettings.syncType === models_1.SyncType.WEBHOOKS;
             if (isWebhookSync) {
-                syncSettings = yield (0, storage_1.getStorage)().getSyncSettings(clientId, crowdinId, 'schedule', provider);
+                syncSettings = (yield (0, storage_1.getStorage)().getSyncSettings(clientId, crowdinId, 'schedule', provider));
                 rootFolder = yield (0, defaults_1.getRootFolder)(config, integration, crowdinClient.client, projectId);
+                if ((_a = integration.syncNewElements) === null || _a === void 0 ? void 0 : _a[provider]) {
+                    newFiles = yield (0, file_snapshot_1.getAllNewFiles)({
+                        config,
+                        integration,
+                        crowdinApiClient: crowdinClient.client,
+                        crowdinId,
+                        integrationCredentials: preparedIntegrationCredentials,
+                        integrationId: clientId,
+                        projectId,
+                        integrationSettings: appSettings,
+                        syncSettings,
+                    });
+                }
             }
         }
-        return { projectId, crowdinClient, preparedIntegrationCredentials, rootFolder, appSettings, syncSettings };
+        return {
+            projectId,
+            crowdinClient,
+            preparedIntegrationCredentials,
+            rootFolder,
+            appSettings,
+            syncSettings,
+            newFiles,
+        };
     });
 }
 exports.prepareWebhookData = prepareWebhookData;
@@ -252,9 +276,9 @@ function updateCrowdinFromWebhookRequest(integration, webhookData, req) {
     var _a, _b;
     return __awaiter(this, void 0, void 0, function* () {
         let filesToSync = [];
-        const { projectId, crowdinClient, preparedIntegrationCredentials, rootFolder, appSettings, syncSettings } = webhookData;
+        const { projectId, crowdinClient, preparedIntegrationCredentials, rootFolder, appSettings, syncSettings, newFiles, } = webhookData;
         if ((_a = integration.webhooks) === null || _a === void 0 ? void 0 : _a.integrationWebhookInterceptor) {
-            filesToSync = yield ((_b = integration.webhooks) === null || _b === void 0 ? void 0 : _b.integrationWebhookInterceptor(projectId, crowdinClient.client, preparedIntegrationCredentials, rootFolder, appSettings, syncSettings, req));
+            filesToSync = yield ((_b = integration.webhooks) === null || _b === void 0 ? void 0 : _b.integrationWebhookInterceptor(projectId, crowdinClient.client, preparedIntegrationCredentials, rootFolder, appSettings, syncSettings, req, newFiles));
         }
         return yield integration.updateCrowdin(projectId, crowdinClient.client, preparedIntegrationCredentials, filesToSync, rootFolder, appSettings);
     });
@@ -296,7 +320,7 @@ function consumer(channel, config, integration) {
                 const data = JSON.parse(msg.content.toString());
                 const urlParam = (_a = integration.webhooks) === null || _a === void 0 ? void 0 : _a.urlParam;
                 const webhookUrlParam = data.query[urlParam];
-                const webhookData = yield prepareWebhookData(config, integration, webhookUrlParam, 'integration');
+                const webhookData = yield prepareWebhookData(config, integration, webhookUrlParam, models_1.Provider.INTEGRATION);
                 yield updateCrowdinFromWebhookRequest(integration, webhookData, data);
             }
             catch (e) {

package/out/views/main.handlebars

@@ -34,6 +34,12 @@
             {{/if}}
         </div>
         <crowdin-simple-integration
+                {{#if syncNewElements.crowdin}}
+                    skip-crowdin-auto-schedule
+                {{/if}}
+                {{#if syncNewElements.integration}}
+                    skip-integration-auto-schedule
+                {{/if}}
                 {{#or withCronSync.crowdin webhooks.crowdin}}
                     crowdin-schedule="true"
                 {{/or}}
@@ -170,7 +176,12 @@
                             <crowdin-p>{{label}}</crowdin-p>
                         {{/if}}
                     {{/if}}
-                    <div style="padding: 8px"></div>
+                    <div
+                        style="padding: 8px"
+                        {{#if dependencySettings}}
+                            data-dependency="{{dependencySettings}}"
+                        {{/if}}
+                    ></div>
                 {{/each}}
             </div>
             <div slot="footer">
@@ -178,6 +189,36 @@
             </div>
         </crowdin-modal>
     {{/if}}
+    {{#or syncNewElements.crowdin syncNewElements.integration}}
+        <crowdin-modal
+                id="confirm-schedule-modal"
+                modal-width="50"
+                modal-title="Synchronization options"
+                close-button-title="Close"
+                close-button="true"
+                body-overflow-unset
+        >
+            <crowdin-checkbox
+                    id="selected-files"
+                    name="selected-files"
+                    label="Selected files"
+                    class="hydrated"
+                    onchange="onChangeAutoSynchronizationOptions(this)"
+            >
+            </crowdin-checkbox>
+            <crowdin-checkbox
+                    id="new-files"
+                    name="new-files"
+                    label="New files"
+                    class="hydrated"
+                    onchange="onChangeAutoSynchronizationOptions(this)"
+            >
+            </crowdin-checkbox>
+            <div slot="footer">
+                <crowdin-button outlined id="save-schedule-sync" onclick="saveScheduleSync()">Save</crowdin-button>
+            </div>
+        </crowdin-modal>
+    {{/or}}
 </body>
 <script type="text/javascript">
     document.body.addEventListener('refreshFilesList', (e) => {
@@ -267,7 +308,7 @@
     function getIntegrationData(hardReload = false, parentId = '', search = '', page = 0) {
         appComponent.setAttribute('is-integration-loading', true);
         checkOrigin()
-            .then(restParams => fetch(`api/integration/data${restParams}&parent_id=${parentId}&search=${search}&page=${page}`))
+            .then(restParams => fetch(`api/integration/data${restParams}&parent_id=${encodeURIComponent(parentId)}&search=${encodeURIComponent(search)}&page=${page}`))
             .then(checkResponse)
             .then((res) => {
                 const files = res.data;
@@ -317,9 +358,18 @@
             .then(checkResponse)
             .then((res) => {
                 if (provider === 'crowdin') {
-                    appComponent.setCrowdinScheduleSync(res);
+                    {{#if syncNewElements.crowdin}}
+                        appComponent.setCrowdinScheduleSync(res, true, true);
+                    {{else}}
+                        appComponent.setCrowdinScheduleSync(res);
+                    {{/if}}
                 } else {
-                    appComponent.setIntegrationScheduleSync(res);
+                    {{#if syncNewElements.integration}}
+                        appComponent.setIntegrationScheduleSync(res, true, true);
+                    {{else}}
+                        appComponent.setIntegrationScheduleSync(res);
+                    {{/if}}
+
                 }
             })
             .catch(e => catchRejection(e, 'Can\'t fetch file progress'));
@@ -521,19 +571,91 @@
     {{/if}}
 
     {{#or withCronSync webhooks}}
+        const scheduleModal = document.getElementById('confirm-schedule-modal');
+        let syncData;
+
         document.body.addEventListener('integrationScheduleSync', setIntegrationScheduleSync);
         document.body.addEventListener('crowdinScheduleSync', setCrowdinScheduleSync);
         document.body.addEventListener('crowdinDisableSync', disableCrowdinSync);
         document.body.addEventListener('integrationDisableSync', disableIntegrationSync);
 
+        async function saveScheduleSync() {
+            const newFile = scheduleModal.querySelector('#new-files').checked || false;
+            const selectedFiles = scheduleModal.querySelector('#selected-files').checked || false;
+
+            const type = scheduleModal.getAttribute('data-type');
+
+            if (type === 'crowdin') {
+                appComponent.setCrowdinScheduleSync(syncData, newFile, selectedFiles);
+                const syncedFiles = await appComponent.getCrowdinScheduleSync(true);
+                appComponent.setAttribute('is-crowdin-loading', true);
+                updateSyncSettings(syncedFiles, 'schedule', 'crowdin');
+            } else if (type === 'integration') {
+                appComponent.setIntegrationScheduleSync(syncData, newFile, selectedFiles);
+                const syncedFiles = await appComponent.getIntegrationScheduleSync(true);
+                appComponent.setAttribute('is-integration-loading', true);
+                updateSyncSettings(syncedFiles, 'schedule', 'integration');
+            }
+
+            scheduleModal.close();
+        }
+
+        function openScheduleModal(type) {
+            const newFile = scheduleModal.querySelector('#new-files')
+            const selectedFiles = scheduleModal.querySelector('#selected-files');
+
+            newFile.checked = false;
+            newFile.value = false;
+            selectedFiles.checked = true;
+            selectedFiles.value = true;
+            scheduleModal.querySelector('#save-schedule-sync').setAttribute('disabled', false);
+            scheduleModal.setAttribute('data-type', type);
+            scheduleModal.open();
+        }
+
+        function onChangeAutoSynchronizationOptions() {
+            const newFiles = document.getElementById('new-files').checked || false;
+            const selectedFiles = document.getElementById('selected-files').checked || false;
+            const buttonSaveScheduleSync = document.getElementById('save-schedule-sync');
+
+            if (newFiles || selectedFiles) {
+                buttonSaveScheduleSync.removeAttribute('disabled');
+            } else {
+                buttonSaveScheduleSync.setAttribute('disabled', true);
+            }
+        }
+
+        async function hasFolder(selectedFiles) {
+            let isFolder;
+            if (Array.isArray(selectedFiles)) {
+                isFolder = selectedFiles.find((file) => file.node_type === folderType || file.node_type === branchType);
+            } else {
+                const files = await appComponent.getCrowdinFilesData();
+                const folders = files.filter((file) => file.node_type === folderType || file.node_type === branchType);
+                isFolder = folders.find((folder) => selectedFiles.hasOwnProperty(folder.id));
+            }
+
+            return isFolder !== undefined;
+        }
+
         async function setIntegrationScheduleSync(e) {
             if (e.detail.length === 0) {
                 showToast('Select templates which will be pushed to Crowdin');
                 return;
             }
-            appComponent.setAttribute('is-integration-loading', true);
-            const syncedFiles = await appComponent.getIntegrationScheduleSync();
 
+            {{#if syncNewElements.integration}}
+                syncData = e.detail;
+                const isFolder = await hasFolder(e.detail);
+                if (isFolder) {
+                    openScheduleModal('integration');
+                    return;
+                }
+
+                appComponent.setIntegrationScheduleSync(e.detail);
+            {{/if}}
+            const syncedFiles = await appComponent.getIntegrationScheduleSync();
+            appComponent.setAttribute('is-integration-loading', true);
             updateSyncSettings(syncedFiles, 'schedule', 'integration');
         }
 
@@ -542,9 +664,19 @@
                 showToast('Select templates which will be pushed to Crowdin');
                 return;
             }
-            appComponent.setAttribute('is-crowdin-loading', true);
-            const syncedFiles = await appComponent.getCrowdinScheduleSync();
 
+            {{#if syncNewElements.crowdin}}
+                syncData = e.detail;
+                const isFolder = await hasFolder(e.detail);
+                if (isFolder) {
+                    openScheduleModal('crowdin');
+                    return;
+                }
+
+                appComponent.setCrowdinScheduleSync(e.detail);
+            {{/if}}
+            const syncedFiles = await appComponent.getCrowdinScheduleSync();
+            appComponent.setAttribute('is-crowdin-loading', true);
             updateSyncSettings(syncedFiles, 'schedule', 'crowdin');
         }
 

package/package.json

@@ -1,11 +1,12 @@
 {
   "name": "@crowdin/app-project-module",
-  "version": "0.28.11",
+  "version": "0.29.0",
   "description": "Module that generates for you all common endpoints for serving standalone Crowdin App",
   "main": "out/index.js",
   "types": "out/index.d.ts",
   "scripts": {
     "build": "tsc -p ./ && cp -R views out && cp -R static out && cp logo.png out && rollup -c rollup.config.mjs",
+    "build-main": "tsc -p ./ && cp -R views out && cp -R static out && cp logo.png out",
     "lint": "eslint --fix \"{src,tests}/**/*.{js,ts}\"",
     "prettier": "prettier --config .prettierrc src/**/*.ts --write",
     "lint-ci": "eslint \"{src,tests}/**/*.{js,ts}\"",
@@ -22,7 +23,9 @@
     "mysql2": "^2.3.3",
     "node-cron": "^3.0.2",
     "pg": "^8.10.0",
+    "redoc-express": "^2.1.0",
     "sqlite3": "^5.1.6",
+    "swagger-jsdoc": "^6.2.8",
     "uuid": "^8.3.2"
   },
   "devDependencies": {
@@ -48,6 +51,7 @@
     "@types/jest": "^29.5.2",
     "@types/node": "^12.20.55",
     "@types/node-cron": "^3.0.7",
+    "@types/swagger-jsdoc": "^6.0.1",
     "@typescript-eslint/eslint-plugin": "^2.3.1",
     "@typescript-eslint/parser": "^2.3.1",
     "eslint": "^6.4.0",