@crowdin/app-project-module 0.102.0 → 0.102.1

package/out/modules/integration/handlers/crowdin-webhook.js

@@ -15,6 +15,7 @@ const logger_1 = require("../../../util/logger");
 const webhooks_1 = require("../util/webhooks");
 const types_1 = require("../types");
 const cron_1 = require("../util/cron");
+const files_1 = require("../util/files");
 function filterSyncFiles(args) {
     var _a, _b;
     return __awaiter(this, void 0, void 0, function* () {
@@ -88,7 +89,7 @@ function handle(config, integration) {
             if (!syncSettings) {
                 return res.status(200).send({ message: 'Sync is not configured' });
             }
-            const syncFileSettings = JSON.parse(syncSettings.files);
+            const syncFileSettings = (0, files_1.prepareSyncFiles)(JSON.parse(syncSettings.files));
             if ((_b = integration.webhooks) === null || _b === void 0 ? void 0 : _b.crowdinWebhookInterceptor) {
                 filesToSync = yield integration.webhooks.crowdinWebhookInterceptor(projectId, crowdinClient.client, rootFolder, appSettings, syncSettings, req.body);
             }

package/out/modules/integration/handlers/sync-settings-save.js

@@ -76,7 +76,7 @@ function handle(config, integration) {
                 else {
                     yield (0, cron_1.createOrUpdateSyncSettings)({
                         req,
-                        files,
+                        files: (0, files_1.prepareSyncFiles)(files),
                         provider,
                     });
                 }

package/out/modules/integration/util/cron.js

@@ -154,7 +154,7 @@ function processSyncSettings({ config, integration, period, syncSettings, }) {
         let projectData;
         let crowdinClient;
         let token;
-        let files = JSON.parse(syncSettings.files);
+        let files = (0, files_1.prepareSyncFiles)(JSON.parse(syncSettings.files));
         let newFiles = [];
         const crowdinCredentials = yield (0, storage_1.getStorage)().getCrowdinCredentials(syncSettings.crowdinId);
         const integrationCredentials = yield (0, storage_1.getStorage)().getIntegrationCredentials(syncSettings.integrationId);
@@ -225,20 +225,27 @@ function processSyncSettings({ config, integration, period, syncSettings, }) {
         context.jwtPayload.context.project_identifier = projectData.identifier;
         const rootFolder = yield (0, defaults_1.getRootFolder)(config, integration, crowdinClient, projectId);
         const apiCredentials = yield (0, connection_1.prepareIntegrationCredentials)(config, integration, integrationCredentials);
-        if (!integration.webhooks &&
+        let currentFileSnapshot = [];
+        const needsSnapshotForNewFiles = !integration.webhooks &&
             ((_a = integration.syncNewElements) === null || _a === void 0 ? void 0 : _a[syncSettings.provider]) &&
-            intConfig[`new-${syncSettings.provider}-files`]) {
+            intConfig[`new-${syncSettings.provider}-files`];
+        const needsIntegrationSnapshot = integration.forcePushSources || (needsSnapshotForNewFiles && syncSettings.provider !== types_1.Provider.CROWDIN);
+        const needsCrowdinSnapshot = needsSnapshotForNewFiles && syncSettings.provider === types_1.Provider.CROWDIN;
+        if (needsIntegrationSnapshot) {
+            currentFileSnapshot = yield (0, snapshot_1.getIntegrationSnapshot)(integration, apiCredentials, intConfig);
+        }
+        else if (needsCrowdinSnapshot) {
+            currentFileSnapshot = yield (0, snapshot_1.getCrowdinSnapshot)(config, integration, crowdinClient, projectData.id, intConfig);
+        }
+        if (needsSnapshotForNewFiles) {
             try {
                 newFiles = yield getAllNewFiles({
-                    config,
-                    integration,
-                    projectData,
-                    syncSettings,
-                    crowdinApiClient: crowdinClient,
                     crowdinId: crowdinCredentials.id,
-                    integrationCredentials: apiCredentials,
                     integrationId: integrationCredentials.id,
+                    projectData,
                     integrationSettings: intConfig,
+                    syncSettings,
+                    currentFileSnapshot,
                 });
             }
             catch (e) {
@@ -331,6 +338,18 @@ function processSyncSettings({ config, integration, period, syncSettings, }) {
                 // eslint-disable-next-line @typescript-eslint/camelcase
                 node_type: file.nodeType || file.node_type }, (file.type ? { type: file.type } : {}))));
             let intFiles = allIntFiles.filter((file) => 'type' in file);
+            if (integration.forcePushSources === true && currentFileSnapshot.length > 0) {
+                const snapshotMap = new Map(currentFileSnapshot.map((f) => [f.id, f]));
+                intFiles = intFiles.map((file) => {
+                    const snapshotFile = snapshotMap.get(file.id);
+                    if (snapshotFile) {
+                        return Object.assign(Object.assign({}, file), { updatedAt: snapshotFile.updatedAt, createdAt: snapshotFile.createdAt });
+                    }
+                    return file;
+                });
+                intFiles = yield (0, files_1.attachFileStatus)(intFiles, syncSettings.integrationId, syncSettings.crowdinId, integration);
+                intFiles = intFiles.filter((file) => file.isUpdated !== false);
+            }
             if (intFiles.length <= 0) {
                 return;
             }
@@ -389,21 +408,13 @@ function processSyncSettings({ config, integration, period, syncSettings, }) {
 function getFileDiff(currentFiles, savedFiles) {
     return currentFiles.filter((x) => !savedFiles.some((x2) => x2.id === x.id));
 }
-function getAllNewFiles(args) {
+function getAllNewFiles({ crowdinId, integrationId, projectData, integrationSettings, syncSettings, currentFileSnapshot, }) {
     return __awaiter(this, void 0, void 0, function* () {
-        const { config, integration, crowdinApiClient, crowdinId, integrationCredentials, integrationId, projectData, integrationSettings, syncSettings, } = args;
-        let currentFileSnapshot = [];
         const fileSnapshotData = yield (0, storage_1.getStorage)().getFilesSnapshot(integrationId, crowdinId, syncSettings.provider);
         const snapshotFiles = (fileSnapshotData === null || fileSnapshotData === void 0 ? void 0 : fileSnapshotData.files) ? JSON.parse(fileSnapshotData.files) : [];
-        if (syncSettings.provider === types_1.Provider.CROWDIN) {
-            currentFileSnapshot = yield (0, snapshot_1.getCrowdinSnapshot)(config, integration, crowdinApiClient, projectData.id, integrationSettings);
-        }
-        else {
-            currentFileSnapshot = yield (0, snapshot_1.getIntegrationSnapshot)(integration, integrationCredentials, integrationSettings);
-        }
         const difference = getFileDiff(currentFileSnapshot, snapshotFiles);
         const onlyFiles = difference.filter((file) => 'type' in file);
-        const synFiles = JSON.parse(syncSettings.files);
+        const synFiles = (0, files_1.prepareSyncFiles)(JSON.parse(syncSettings.files));
         if (syncSettings.provider === types_1.Provider.INTEGRATION) {
             if (integrationSettings[`new-${syncSettings.provider}-files`]) {
                 return onlyFiles;

package/out/modules/integration/util/files.d.ts

@@ -4,6 +4,7 @@ import Crowdin from '@crowdin/crowdin-api-client';
 import { SourceFilesModel } from '@crowdin/crowdin-api-client';
 export declare function skipFilesByRegex(files: TreeItem[] | undefined, skipIntegrationNodes?: SkipIntegrationNodes): TreeItem[];
 export declare function expandFilesTree(nodes: IntegrationFile[], req: IntegrationRequest, integration: IntegrationLogic, job?: JobClient): Promise<IntegrationFile[]>;
+export declare function attachFileStatus(files: any[], clientId: string, crowdinId: string, integration: IntegrationLogic): Promise<any[]>;
 export declare function isExtendedResultType<T>(data?: T | ExtendedResult<T>): data is ExtendedResult<T>;
 export declare function markUnsyncedFiles({ integrationId, crowdinId, client, files, }: {
     integrationId: string;
@@ -17,6 +18,8 @@ export declare function getExcludedTargetLanguages({ client, projectId, language
     languages: string[];
 }): Promise<string[]>;
 export declare function filterLanguages(request: UpdateIntegrationRequest, files: SourceFilesModel.File[]): UpdateIntegrationRequest;
+export declare function uniqueFileLanguages(files: UpdateIntegrationRequest): UpdateIntegrationRequest;
+export declare function prepareSyncFiles(files: any): any;
 export declare function buildPath(file: TreeItem, files: TreeItem[]): string;
 export declare function getParentPaths(filePath: string): Set<string>;
 export declare function getParentFiles(allFiles: TreeItem[], fileCollection: TreeItem[]): TreeItem[];

package/out/modules/integration/util/files.js

@@ -12,7 +12,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.filterSyncedData = exports.updateSyncedData = exports.filterFilesByPath = exports.filterFilesByPatterns = exports.getParentFiles = exports.getParentPaths = exports.buildPath = exports.filterLanguages = exports.getExcludedTargetLanguages = exports.markUnsyncedFiles = exports.isExtendedResultType = exports.expandFilesTree = exports.skipFilesByRegex = void 0;
+exports.filterSyncedData = exports.updateSyncedData = exports.filterFilesByPath = exports.filterFilesByPatterns = exports.getParentFiles = exports.getParentPaths = exports.buildPath = exports.prepareSyncFiles = exports.uniqueFileLanguages = exports.filterLanguages = exports.getExcludedTargetLanguages = exports.markUnsyncedFiles = exports.isExtendedResultType = exports.attachFileStatus = exports.expandFilesTree = exports.skipFilesByRegex = void 0;
 const types_1 = require("../types");
 const types_2 = require("./types");
 const storage_1 = require("../../../storage");
@@ -75,44 +75,57 @@ function expandFilesTree(nodes, req, integration, job) {
             integration.forcePushSources === true;
         if (needsFileStatus) {
             const { crowdinId, clientId } = req.crowdinContext;
-            const syncedData = yield (0, storage_1.getStorage)().getSyncedData(clientId, crowdinId, types_1.Provider.INTEGRATION);
-            const syncedFiles = (syncedData === null || syncedData === void 0 ? void 0 : syncedData.files) ? JSON.parse(syncedData.files) : [];
-            const lastSyncTimestamp = (syncedData === null || syncedData === void 0 ? void 0 : syncedData.updatedAt) ? Number(syncedData.updatedAt) : null;
-            return files.map((file) => {
-                var _a, _b, _c, _d, _e, _f;
-                if (!file.type) {
-                    return file;
-                }
-                const syncedFile = syncedFiles.find((syncedItem) => syncedItem.id === file.id);
-                const notSynced = ((_b = (_a = integration.filtering) === null || _a === void 0 ? void 0 : _a.integrationFileStatus) === null || _b === void 0 ? void 0 : _b.notSynced) === true ? !syncedFile : false;
-                const fileSyncTimestamp = (syncedFile === null || syncedFile === void 0 ? void 0 : syncedFile.syncedAt) ? Number(syncedFile.syncedAt) : lastSyncTimestamp;
-                const isNew = ((_d = (_c = integration.filtering) === null || _c === void 0 ? void 0 : _c.integrationFileStatus) === null || _d === void 0 ? void 0 : _d.isNew) === true
-                    ? !syncedFile &&
-                        fileSyncTimestamp &&
-                        file.createdAt &&
-                        (typeof file.createdAt === 'string' ? new Date(file.createdAt).getTime() : file.createdAt) >
-                            fileSyncTimestamp
-                    : false;
-                const shouldCalculateIsUpdated = ((_f = (_e = integration.filtering) === null || _e === void 0 ? void 0 : _e.integrationFileStatus) === null || _f === void 0 ? void 0 : _f.isUpdated) === true ||
-                    integration.forcePushSources === true;
-                const isUpdated = file.isUpdated !== undefined
-                    ? file.isUpdated
-                    : shouldCalculateIsUpdated
-                        ? syncedFile &&
-                            fileSyncTimestamp &&
-                            file.updatedAt &&
-                            (typeof file.updatedAt === 'string' ? new Date(file.updatedAt).getTime() : file.updatedAt) >
-                                fileSyncTimestamp
-                        : false;
-                return Object.assign(Object.assign({}, file), { isNew,
-                    notSynced,
-                    isUpdated });
-            });
+            return yield attachFileStatus(files, clientId, crowdinId, integration);
         }
         return files;
     });
 }
 exports.expandFilesTree = expandFilesTree;
+function attachFileStatus(files, clientId, crowdinId, integration) {
+    var _a, _b, _c, _d, _e, _f;
+    return __awaiter(this, void 0, void 0, function* () {
+        const needsFileStatus = ((_b = (_a = integration.filtering) === null || _a === void 0 ? void 0 : _a.integrationFileStatus) === null || _b === void 0 ? void 0 : _b.isNew) ||
+            ((_d = (_c = integration.filtering) === null || _c === void 0 ? void 0 : _c.integrationFileStatus) === null || _d === void 0 ? void 0 : _d.isUpdated) ||
+            ((_f = (_e = integration.filtering) === null || _e === void 0 ? void 0 : _e.integrationFileStatus) === null || _f === void 0 ? void 0 : _f.notSynced) ||
+            integration.forcePushSources === true;
+        if (!needsFileStatus) {
+            return files;
+        }
+        const syncedData = yield (0, storage_1.getStorage)().getSyncedData(clientId, crowdinId, types_1.Provider.INTEGRATION);
+        const syncedFiles = (syncedData === null || syncedData === void 0 ? void 0 : syncedData.files) ? JSON.parse(syncedData.files) : [];
+        const lastSyncTimestamp = (syncedData === null || syncedData === void 0 ? void 0 : syncedData.updatedAt) ? Number(syncedData.updatedAt) : null;
+        return files.map((file) => {
+            var _a, _b, _c, _d, _e, _f;
+            if (!file.type) {
+                return file;
+            }
+            const syncedFile = syncedFiles.find((syncedItem) => syncedItem.id === file.id);
+            const notSynced = ((_b = (_a = integration.filtering) === null || _a === void 0 ? void 0 : _a.integrationFileStatus) === null || _b === void 0 ? void 0 : _b.notSynced) === true ? !syncedFile : false;
+            const fileSyncTimestamp = (syncedFile === null || syncedFile === void 0 ? void 0 : syncedFile.syncedAt) ? Number(syncedFile.syncedAt) : lastSyncTimestamp;
+            const isNew = ((_d = (_c = integration.filtering) === null || _c === void 0 ? void 0 : _c.integrationFileStatus) === null || _d === void 0 ? void 0 : _d.isNew) === true
+                ? !syncedFile &&
+                    fileSyncTimestamp &&
+                    file.createdAt &&
+                    (typeof file.createdAt === 'string' ? new Date(file.createdAt).getTime() : file.createdAt) >
+                        fileSyncTimestamp
+                : false;
+            const shouldCalculateIsUpdated = ((_f = (_e = integration.filtering) === null || _e === void 0 ? void 0 : _e.integrationFileStatus) === null || _f === void 0 ? void 0 : _f.isUpdated) === true || integration.forcePushSources === true;
+            const isUpdated = file.isUpdated !== undefined
+                ? file.isUpdated
+                : shouldCalculateIsUpdated
+                    ? syncedFile &&
+                        fileSyncTimestamp &&
+                        file.updatedAt &&
+                        (typeof file.updatedAt === 'string' ? new Date(file.updatedAt).getTime() : file.updatedAt) >
+                            fileSyncTimestamp
+                    : false;
+            return Object.assign(Object.assign({}, file), { isNew,
+                notSynced,
+                isUpdated });
+        });
+    });
+}
+exports.attachFileStatus = attachFileStatus;
 function isExtendedResultType(data) {
     const dataTyped = data;
     return !!dataTyped && !Array.isArray(dataTyped);
@@ -182,6 +195,29 @@ function filterLanguages(request, files) {
     return result;
 }
 exports.filterLanguages = filterLanguages;
+function uniqueFileLanguages(files) {
+    const result = {};
+    for (const fileId in files) {
+        const languages = files[fileId];
+        result[fileId] = Array.isArray(languages) ? [...new Set(languages)] : languages;
+    }
+    return result;
+}
+exports.uniqueFileLanguages = uniqueFileLanguages;
+function prepareSyncFiles(files) {
+    // If it's an array (Provider.INTEGRATION format), return as is
+    if (Array.isArray(files)) {
+        return files;
+    }
+    if (files && typeof files === 'object') {
+        const hasLanguageArrays = Object.values(files).some((value) => Array.isArray(value));
+        if (hasLanguageArrays) {
+            return uniqueFileLanguages(files);
+        }
+    }
+    return files;
+}
+exports.prepareSyncFiles = prepareSyncFiles;
 function buildPath(file, files) {
     const filePath = `/${file.name}`;
     if (!file.parentId) {

package/out/modules/integration/util/types.d.ts

@@ -1,6 +1,5 @@
-import Crowdin, { ProjectsGroupsModel } from '@crowdin/crowdin-api-client';
-import { Config } from '../../../types';
-import { IntegrationLogic, IntegrationSyncSettings } from '../types';
+import { ProjectsGroupsModel } from '@crowdin/crowdin-api-client';
+import { IntegrationSyncSettings, TreeItem } from '../types';
 import { ResponseObject } from '@crowdin/crowdin-api-client/out/core';
 import { TranslationsModel } from '@crowdin/crowdin-api-client/out/translations';
 import { AxiosError } from 'axios';
@@ -69,15 +68,12 @@ export type UpdateJobProgress = ({ progress, status, info, data, attempt, errors
     isCanceled: boolean;
 }>;
 export interface GetAllNewFilesArgs {
-    config: Config;
-    integration: IntegrationLogic;
-    crowdinApiClient: Crowdin;
     crowdinId: string;
-    integrationCredentials: any;
     integrationId: string;
     projectData: ProjectsGroupsModel.ProjectSettings;
     integrationSettings: any;
     syncSettings: IntegrationSyncSettings;
+    currentFileSnapshot: TreeItem[];
 }
 export type SaveUploadedFileTranslation = ({ fileId, translationParams, }: {
     fileId: number;

package/out/modules/integration/util/webhooks.js

@@ -45,6 +45,7 @@ const connection_1 = require("../../../util/connection");
 const defaults_1 = require("./defaults");
 const index_1 = require("../../../util/index");
 const logger_1 = require("../../../util/logger");
+const files_1 = require("./files");
 const prefetchCount = 10;
 const forceProcessDelay = 10000;
 const maxReconnectAttempts = 5;
@@ -315,7 +316,7 @@ function updateCrowdinFromWebhookRequest(args) {
         const { integration, webhookData, req } = args;
         let filesToSync = [];
         const { projectId, crowdinClient, preparedIntegrationCredentials, rootFolder, appSettings, syncSettings } = webhookData;
-        const syncFiles = (syncSettings === null || syncSettings === void 0 ? void 0 : syncSettings.files) ? JSON.parse(syncSettings.files) : [];
+        const syncFiles = (syncSettings === null || syncSettings === void 0 ? void 0 : syncSettings.files) ? (0, files_1.prepareSyncFiles)(JSON.parse(syncSettings.files)) : [];
         if ((_a = integration.webhooks) === null || _a === void 0 ? void 0 : _a.integrationWebhookInterceptor) {
             filesToSync = yield ((_b = integration.webhooks) === null || _b === void 0 ? void 0 : _b.integrationWebhookInterceptor(projectId, crowdinClient.client, preparedIntegrationCredentials, rootFolder, appSettings, syncSettings, req));
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@crowdin/app-project-module",
-  "version": "0.102.0",
+  "version": "0.102.1",
   "description": "Module that generates for you all common endpoints for serving standalone Crowdin App",
   "main": "out/index.js",
   "types": "out/index.d.ts",