@crowdin/app-project-module 2.9.1 → 2.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/index.d.ts +2 -0
- package/out/index.js +11 -4
- package/out/modules/integration/util/context.d.ts +11 -0
- package/out/modules/integration/util/context.js +17 -0
- package/out/modules/integration/util/cron.js +70 -63
- package/out/modules/integration/util/job.js +88 -80
- package/out/modules/integration/util/webhooks.d.ts +11 -0
- package/out/modules/integration/util/webhooks.js +35 -56
- package/out/static/css/styles.css +2 -2
- package/out/static/ui/about.bundle.js.map +1 -1
- package/out/static/ui/error.bundle.js.map +1 -1
- package/out/static/ui/form.bundle.js +87 -120
- package/out/static/ui/form.bundle.js.map +1 -1
- package/out/static/ui/install.bundle.js.map +1 -1
- package/out/static/ui/login.bundle.js.map +1 -1
- package/out/static/ui/main.bundle.js.map +1 -1
- package/out/static/ui/oauth.bundle.js.map +1 -1
- package/out/static/ui/subscription.bundle.js.map +1 -1
- package/out/util/cron.d.ts +0 -2
- package/out/util/cron.js +18 -56
- package/out/util/logger.d.ts +23 -1
- package/out/util/logger.js +36 -38
- package/package.json +11 -6
package/out/index.d.ts
CHANGED
|
@@ -5,6 +5,8 @@ import { getRequestCredentialsMasker, postRequestCredentialsMasker, maskKey } fr
|
|
|
5
5
|
export { getRequestCredentialsMasker, postRequestCredentialsMasker, maskKey };
|
|
6
6
|
export * from './types';
|
|
7
7
|
export { Cron } from './util/cron';
|
|
8
|
+
/** for app code that fans work out inside one request or job: each branch opens its own log scope */
|
|
9
|
+
export { runWithContext } from '@crowdin/logs-formatter';
|
|
8
10
|
export { express };
|
|
9
11
|
export { reportInsight as reportAdvisorInsight } from './modules/advisor-inspector/report-insight';
|
|
10
12
|
export declare const metadataStore: CrowdinMetadataStore;
|
package/out/index.js
CHANGED
|
@@ -48,7 +48,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
48
48
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
49
49
|
};
|
|
50
50
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
51
|
-
exports.metadataStore = exports.reportAdvisorInsight = exports.express = exports.maskKey = exports.postRequestCredentialsMasker = exports.getRequestCredentialsMasker = void 0;
|
|
51
|
+
exports.metadataStore = exports.reportAdvisorInsight = exports.express = exports.runWithContext = exports.maskKey = exports.postRequestCredentialsMasker = exports.getRequestCredentialsMasker = void 0;
|
|
52
52
|
exports.createApp = createApp;
|
|
53
53
|
exports.addCrowdinEndpoints = addCrowdinEndpoints;
|
|
54
54
|
const logsFormatter = __importStar(require("@crowdin/logs-formatter"));
|
|
@@ -111,6 +111,9 @@ const automationAction = __importStar(require("./modules/automation-action"));
|
|
|
111
111
|
const authGuard = __importStar(require("./modules/auth-guard"));
|
|
112
112
|
const subscription_1 = require("./util/subscription");
|
|
113
113
|
__exportStar(require("./types"), exports);
|
|
114
|
+
/** for app code that fans work out inside one request or job: each branch opens its own log scope */
|
|
115
|
+
var logs_formatter_1 = require("@crowdin/logs-formatter");
|
|
116
|
+
Object.defineProperty(exports, "runWithContext", { enumerable: true, get: function () { return logs_formatter_1.runWithContext; } });
|
|
114
117
|
var report_insight_1 = require("./modules/advisor-inspector/report-insight");
|
|
115
118
|
Object.defineProperty(exports, "reportAdvisorInsight", { enumerable: true, get: function () { return report_insight_1.reportInsight; } });
|
|
116
119
|
exports.metadataStore = {
|
|
@@ -159,6 +162,12 @@ function addCrowdinEndpoints(app, clientConfig) {
|
|
|
159
162
|
logger.initialize(config);
|
|
160
163
|
storage.initialize(config);
|
|
161
164
|
cron.initialize(config);
|
|
165
|
+
// Mounted first, so that even migration and body-parsing failures are attributed to their request
|
|
166
|
+
if (!config.disableLogsFormatter) {
|
|
167
|
+
logsFormatter.setup({ appIdentifier: config.identifier });
|
|
168
|
+
logsFormatter.registerSanitizer(log_sanitizer_1.crowdinLogSanitizer);
|
|
169
|
+
app.use(logsFormatter.contextResolverMiddleware());
|
|
170
|
+
}
|
|
162
171
|
// Middleware to ensure delayed migration before handling requests
|
|
163
172
|
app.use((req, res, next) => __awaiter(this, void 0, void 0, function* () {
|
|
164
173
|
try {
|
|
@@ -185,9 +194,7 @@ function addCrowdinEndpoints(app, clientConfig) {
|
|
|
185
194
|
}
|
|
186
195
|
});
|
|
187
196
|
if (!config.disableLogsFormatter) {
|
|
188
|
-
|
|
189
|
-
logsFormatter.registerSanitizer(log_sanitizer_1.crowdinLogSanitizer);
|
|
190
|
-
app.use(logsFormatter.contextResolverMiddleware());
|
|
197
|
+
// stays after body parsing: it serves POST /_/log from req.body
|
|
191
198
|
app.use(logsFormatter.expressMiddleware());
|
|
192
199
|
}
|
|
193
200
|
app.use('/assets', (0, static_files_1.serveStatic)(config, config.assetsPath || 'static'));
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { CrowdinContextInfo, CrowdinCredentials } from '../../../types';
|
|
2
|
+
import { IntegrationCredentials } from '../types';
|
|
3
|
+
/**
|
|
4
|
+
* Background work (crons, webhooks, job re-runs) has no JWT - this builds the equivalent context
|
|
5
|
+
* from stored credentials, so the same consumers (logging, client preparation) work either way.
|
|
6
|
+
*/
|
|
7
|
+
export declare function crowdinContextInfoFromCredentials({ projectId, crowdinCredentials, integrationCredentials, }: {
|
|
8
|
+
projectId: number;
|
|
9
|
+
crowdinCredentials: CrowdinCredentials;
|
|
10
|
+
integrationCredentials?: IntegrationCredentials;
|
|
11
|
+
}): CrowdinContextInfo;
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.crowdinContextInfoFromCredentials = crowdinContextInfoFromCredentials;
|
|
4
|
+
/**
|
|
5
|
+
* Background work (crons, webhooks, job re-runs) has no JWT - this builds the equivalent context
|
|
6
|
+
* from stored credentials, so the same consumers (logging, client preparation) work either way.
|
|
7
|
+
*/
|
|
8
|
+
function crowdinContextInfoFromCredentials({ projectId, crowdinCredentials, integrationCredentials, }) {
|
|
9
|
+
return Object.assign({ jwtPayload: {
|
|
10
|
+
context: {
|
|
11
|
+
project_id: projectId,
|
|
12
|
+
organization_id: crowdinCredentials.organizationId,
|
|
13
|
+
organization_domain: crowdinCredentials.domain,
|
|
14
|
+
user_id: crowdinCredentials.userId,
|
|
15
|
+
},
|
|
16
|
+
}, crowdinId: crowdinCredentials.id }, ((integrationCredentials === null || integrationCredentials === void 0 ? void 0 : integrationCredentials.id) && { clientId: integrationCredentials.id }));
|
|
17
|
+
}
|
|
@@ -15,6 +15,7 @@ exports.filesCron = filesCron;
|
|
|
15
15
|
exports.filterFilesFromIntegrationRequest = filterFilesFromIntegrationRequest;
|
|
16
16
|
exports.createOrUpdateSyncSettings = createOrUpdateSyncSettings;
|
|
17
17
|
exports.removeFinishedJobs = removeFinishedJobs;
|
|
18
|
+
const logs_formatter_1 = require("@crowdin/logs-formatter");
|
|
18
19
|
const storage_1 = require("../../../storage");
|
|
19
20
|
const util_1 = require("../../../util");
|
|
20
21
|
const token_1 = require("../../../util/app-functions/token");
|
|
@@ -22,6 +23,7 @@ const connection_1 = require("../../../util/connection");
|
|
|
22
23
|
const logger_1 = require("../../../util/logger");
|
|
23
24
|
const subscription_1 = require("../../../util/subscription");
|
|
24
25
|
const types_1 = require("../types");
|
|
26
|
+
const context_1 = require("./context");
|
|
25
27
|
const defaults_1 = require("./defaults");
|
|
26
28
|
const files_1 = require("./files");
|
|
27
29
|
const job_1 = require("./job");
|
|
@@ -30,52 +32,62 @@ const types_2 = require("./types");
|
|
|
30
32
|
// Maximum number of organizations processed concurrently during a cron run.
|
|
31
33
|
// Keeps DB connection and API usage bounded regardless of how many orgs are registered.
|
|
32
34
|
const CRON_ORG_CONCURRENCY = (0, util_1.getEnvInt)('CRON_ORG_CONCURRENCY', 5);
|
|
35
|
+
function runJobForOrganization(_a) {
|
|
36
|
+
return __awaiter(this, arguments, void 0, function* ({ config, integration, job, crowdinCredentials, }) {
|
|
37
|
+
const { token, client: crowdinClient } = yield (0, connection_1.prepareCrowdinClient)({
|
|
38
|
+
config,
|
|
39
|
+
credentials: crowdinCredentials,
|
|
40
|
+
autoRenew: true,
|
|
41
|
+
});
|
|
42
|
+
const { expired } = yield (0, subscription_1.checkSubscription)({
|
|
43
|
+
config,
|
|
44
|
+
token,
|
|
45
|
+
organization: crowdinCredentials.id,
|
|
46
|
+
accountType: crowdinCredentials.type,
|
|
47
|
+
});
|
|
48
|
+
if (expired) {
|
|
49
|
+
(0, logger_1.log)(`Subscription expired. Skipping job [${job.expression}] for organization ${crowdinCredentials.id}`);
|
|
50
|
+
return;
|
|
51
|
+
}
|
|
52
|
+
const integrationCredentialsList = yield (0, storage_1.getStorage)().getAllIntegrationCredentials(crowdinCredentials.id);
|
|
53
|
+
const allIntegrationConfigs = yield (0, storage_1.getStorage)().getAllIntegrationConfigs(crowdinCredentials.id);
|
|
54
|
+
for (const integrationCredentials of integrationCredentialsList) {
|
|
55
|
+
const integrationConfig = allIntegrationConfigs.find(({ integrationId }) => integrationId === integrationCredentials.id);
|
|
56
|
+
const projectId = (0, token_1.getProjectId)(integrationCredentials.id);
|
|
57
|
+
(0, logger_1.setLogCrowdinContext)((0, context_1.crowdinContextInfoFromCredentials)({ projectId, crowdinCredentials, integrationCredentials }));
|
|
58
|
+
const credentials = yield (0, connection_1.prepareIntegrationCredentials)(config, integration, integrationCredentials);
|
|
59
|
+
const rootFolder = yield (0, defaults_1.getRootFolder)(config, integration, crowdinClient, projectId);
|
|
60
|
+
const intConfig = (integrationConfig === null || integrationConfig === void 0 ? void 0 : integrationConfig.config) ? JSON.parse(integrationConfig.config) : undefined;
|
|
61
|
+
(0, logger_1.log)(`Executing task for cron job with expression [${job.expression}] for project ${projectId}`);
|
|
62
|
+
yield job.task({
|
|
63
|
+
projectId,
|
|
64
|
+
client: crowdinClient,
|
|
65
|
+
credentials,
|
|
66
|
+
rootFolder,
|
|
67
|
+
settings: intConfig,
|
|
68
|
+
});
|
|
69
|
+
(0, logger_1.log)(`Task for cron job with expression [${job.expression}] for project ${projectId} completed`);
|
|
70
|
+
}
|
|
71
|
+
});
|
|
72
|
+
}
|
|
33
73
|
function runJob(_a) {
|
|
34
74
|
return __awaiter(this, arguments, void 0, function* ({ config, integration, job, }) {
|
|
35
75
|
(0, logger_1.log)(`Starting cron job with expression [${job.expression}]`);
|
|
36
76
|
const crowdinCredentialsList = yield (0, storage_1.getStorage)().getAllCrowdinCredentials();
|
|
37
77
|
for (let i = 0; i < crowdinCredentialsList.length; i += CRON_ORG_CONCURRENCY) {
|
|
38
78
|
const batch = crowdinCredentialsList.slice(i, i + CRON_ORG_CONCURRENCY);
|
|
39
|
-
const results = yield Promise.allSettled(
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
autoRenew: true,
|
|
44
|
-
});
|
|
45
|
-
const { expired } = yield (0, subscription_1.checkSubscription)({
|
|
46
|
-
config,
|
|
47
|
-
token,
|
|
48
|
-
organization: crowdinCredentials.id,
|
|
49
|
-
accountType: crowdinCredentials.type,
|
|
50
|
-
});
|
|
51
|
-
if (expired) {
|
|
52
|
-
(0, logger_1.log)(`Subscription expired. Skipping job [${job.expression}] for organization ${crowdinCredentials.id}`);
|
|
53
|
-
return;
|
|
54
|
-
}
|
|
55
|
-
const integrationCredentialsList = yield (0, storage_1.getStorage)().getAllIntegrationCredentials(crowdinCredentials.id);
|
|
56
|
-
const allIntegrationConfigs = yield (0, storage_1.getStorage)().getAllIntegrationConfigs(crowdinCredentials.id);
|
|
57
|
-
for (const integrationCredentials of integrationCredentialsList) {
|
|
58
|
-
const integrationConfig = allIntegrationConfigs.find(({ integrationId }) => integrationId === integrationCredentials.id);
|
|
59
|
-
const projectId = (0, token_1.getProjectId)(integrationCredentials.id);
|
|
60
|
-
const credentials = yield (0, connection_1.prepareIntegrationCredentials)(config, integration, integrationCredentials);
|
|
61
|
-
const rootFolder = yield (0, defaults_1.getRootFolder)(config, integration, crowdinClient, projectId);
|
|
62
|
-
const intConfig = (integrationConfig === null || integrationConfig === void 0 ? void 0 : integrationConfig.config) ? JSON.parse(integrationConfig.config) : undefined;
|
|
63
|
-
(0, logger_1.log)(`Executing task for cron job with expression [${job.expression}] for project ${projectId}`);
|
|
64
|
-
yield job.task({
|
|
65
|
-
projectId,
|
|
66
|
-
client: crowdinClient,
|
|
67
|
-
credentials,
|
|
68
|
-
rootFolder,
|
|
69
|
-
settings: intConfig,
|
|
70
|
-
});
|
|
71
|
-
(0, logger_1.log)(`Task for cron job with expression [${job.expression}] for project ${projectId} completed`);
|
|
72
|
-
}
|
|
73
|
-
})));
|
|
74
|
-
for (const result of results) {
|
|
79
|
+
const results = yield Promise.allSettled(
|
|
80
|
+
// per-organization scope, for the same reason processSyncSettings opens one per branch
|
|
81
|
+
batch.map((crowdinCredentials) => (0, logs_formatter_1.runWithContext)((0, logger_1.logContext)(crowdinCredentials), () => runJobForOrganization({ config, integration, job, crowdinCredentials }))));
|
|
82
|
+
results.forEach((result, index) => {
|
|
75
83
|
if (result.status === 'rejected') {
|
|
76
|
-
|
|
84
|
+
// the rejection landed here, outside the branch's scope, so reopen its tenant
|
|
85
|
+
(0, logs_formatter_1.runWithContext)((0, logger_1.logContext)(batch[index]), () => {
|
|
86
|
+
(0, logger_1.log)(`Cron job with expression [${job.expression}] failed for organization '${batch[index].id}'`);
|
|
87
|
+
(0, logger_1.logError)(result.reason);
|
|
88
|
+
});
|
|
77
89
|
}
|
|
78
|
-
}
|
|
90
|
+
});
|
|
79
91
|
}
|
|
80
92
|
(0, logger_1.log)(`Cron job with expression [${job.expression}] completed`);
|
|
81
93
|
});
|
|
@@ -146,22 +158,27 @@ function filesCron(_a) {
|
|
|
146
158
|
const syncSettingsList = yield (0, storage_1.getStorage)().getSyncSettingsBySchedule('schedule', period);
|
|
147
159
|
const crowdinSyncSettings = syncSettingsList.filter((syncSettings) => syncSettings.provider === types_1.Provider.CROWDIN);
|
|
148
160
|
const integrationSyncSettings = syncSettingsList.filter((syncSettings) => syncSettings.provider === types_1.Provider.INTEGRATION);
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
if (result.status === 'rejected') {
|
|
152
|
-
(0, logger_1.logError)(result.reason);
|
|
153
|
-
}
|
|
154
|
-
});
|
|
155
|
-
const integrationResults = yield Promise.allSettled(integrationSyncSettings.map((syncSettings) => processSyncSettings({ config, integration, period, syncSettings })));
|
|
156
|
-
integrationResults.forEach((result) => {
|
|
157
|
-
if (result.status === 'rejected') {
|
|
158
|
-
(0, logger_1.logError)(result.reason);
|
|
159
|
-
}
|
|
160
|
-
});
|
|
161
|
+
yield Promise.all(crowdinSyncSettings.map((syncSettings) => processSyncSettings({ config, integration, period, syncSettings })));
|
|
162
|
+
yield Promise.all(integrationSyncSettings.map((syncSettings) => processSyncSettings({ config, integration, period, syncSettings })));
|
|
161
163
|
(0, logger_1.log)(`Files cron job with period [${period}] completed`);
|
|
162
164
|
});
|
|
163
165
|
}
|
|
164
|
-
|
|
166
|
+
/**
|
|
167
|
+
* Each branch of the fan-out gets a scope of its own: the tick's scope is shared by every tenant
|
|
168
|
+
* due for sync, so a context written there stamps the other tenants' log lines.
|
|
169
|
+
*/
|
|
170
|
+
function processSyncSettings(params) {
|
|
171
|
+
return (0, logs_formatter_1.runWithContext)({}, () => __awaiter(this, void 0, void 0, function* () {
|
|
172
|
+
try {
|
|
173
|
+
yield processSyncSettingsScoped(params);
|
|
174
|
+
}
|
|
175
|
+
catch (e) {
|
|
176
|
+
(0, logger_1.log)(`Files sync failed for organization '${params.syncSettings.crowdinId}'`);
|
|
177
|
+
(0, logger_1.logError)(e);
|
|
178
|
+
}
|
|
179
|
+
}));
|
|
180
|
+
}
|
|
181
|
+
function processSyncSettingsScoped(_a) {
|
|
165
182
|
return __awaiter(this, arguments, void 0, function* ({ config, integration, period, syncSettings, }) {
|
|
166
183
|
var _b, _c, _d, _e;
|
|
167
184
|
let projectData;
|
|
@@ -182,18 +199,8 @@ function processSyncSettings(_a) {
|
|
|
182
199
|
return;
|
|
183
200
|
}
|
|
184
201
|
const projectId = (0, token_1.getProjectId)(integrationCredentials.id);
|
|
185
|
-
const context = {
|
|
186
|
-
|
|
187
|
-
context: {
|
|
188
|
-
project_id: projectId,
|
|
189
|
-
organization_id: crowdinCredentials.organizationId,
|
|
190
|
-
organization_domain: crowdinCredentials.domain,
|
|
191
|
-
user_id: crowdinCredentials.userId,
|
|
192
|
-
},
|
|
193
|
-
},
|
|
194
|
-
crowdinId: crowdinCredentials.id,
|
|
195
|
-
clientId: integrationCredentials.id,
|
|
196
|
-
};
|
|
202
|
+
const context = (0, context_1.crowdinContextInfoFromCredentials)({ projectId, crowdinCredentials, integrationCredentials });
|
|
203
|
+
(0, logger_1.setLogCrowdinContext)(context);
|
|
197
204
|
const logInfo = (0, logger_1.withContext)(context);
|
|
198
205
|
try {
|
|
199
206
|
const preparedCrowdinClient = yield (0, connection_1.prepareCrowdinClient)({
|
|
@@ -11,11 +11,13 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
11
11
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
12
|
exports.runAsJob = runAsJob;
|
|
13
13
|
exports.reRunInProgressJobs = reRunInProgressJobs;
|
|
14
|
+
const logs_formatter_1 = require("@crowdin/logs-formatter");
|
|
14
15
|
const storage_1 = require("../../../storage");
|
|
15
16
|
const token_1 = require("../../../util/app-functions/token");
|
|
16
17
|
const connection_1 = require("../../../util/connection");
|
|
17
18
|
const util_1 = require("../../../util");
|
|
18
19
|
const logger_1 = require("../../../util/logger");
|
|
20
|
+
const context_1 = require("./context");
|
|
19
21
|
const cron_1 = require("./cron");
|
|
20
22
|
const defaults_1 = require("./defaults");
|
|
21
23
|
const types_1 = require("./types");
|
|
@@ -30,7 +32,10 @@ const blockingJobs = {
|
|
|
30
32
|
const maxAttempts = 3;
|
|
31
33
|
// Maximum number of in-progress jobs retried concurrently on startup.
|
|
32
34
|
const RERUN_JOB_CONCURRENCY = (0, util_1.getEnvInt)('RERUN_JOB_CONCURRENCY', 5);
|
|
33
|
-
|
|
35
|
+
// Maximum total etag entries held in the in-memory translation cache.
|
|
36
|
+
const TRANSLATION_CACHE_MAX = (0, util_1.getEnvInt)('TRANSLATION_CACHE_SIZE', 50000);
|
|
37
|
+
let store = {};
|
|
38
|
+
let storeSize = 0;
|
|
34
39
|
function runAsJob(_a) {
|
|
35
40
|
return __awaiter(this, arguments, void 0, function* ({ integrationId, crowdinId, type, title, payload, res, projectId, client, jobType, jobStoreType, responseData, jobCallback, onError, reRunJobId, forcePushTranslations, initiatedBy, }) {
|
|
36
41
|
let jobId;
|
|
@@ -104,22 +109,22 @@ function runAsJob(_a) {
|
|
|
104
109
|
yield storage.saveTranslationCache({ integrationId, crowdinId, fileId, languageId });
|
|
105
110
|
}
|
|
106
111
|
else {
|
|
112
|
+
if (storeSize >= TRANSLATION_CACHE_MAX) {
|
|
113
|
+
(0, logger_1.log)(`Translation etag cache reached limit of ${TRANSLATION_CACHE_MAX} entries, clearing`);
|
|
114
|
+
store = {};
|
|
115
|
+
storeSize = 0;
|
|
116
|
+
}
|
|
107
117
|
if (!store[integrationId]) {
|
|
108
118
|
store[integrationId] = {};
|
|
109
119
|
}
|
|
110
120
|
if (!store[integrationId][crowdinId]) {
|
|
111
|
-
store[integrationId] = {
|
|
112
|
-
[crowdinId]: {},
|
|
113
|
-
};
|
|
121
|
+
store[integrationId][crowdinId] = {};
|
|
114
122
|
}
|
|
115
123
|
if (!store[integrationId][crowdinId][fileId]) {
|
|
116
|
-
store[integrationId] = {
|
|
117
|
-
[crowdinId]: {
|
|
118
|
-
[fileId]: {},
|
|
119
|
-
},
|
|
120
|
-
};
|
|
124
|
+
store[integrationId][crowdinId][fileId] = {};
|
|
121
125
|
}
|
|
122
126
|
store[integrationId][crowdinId][fileId][languageId] = {};
|
|
127
|
+
storeSize++;
|
|
123
128
|
}
|
|
124
129
|
}
|
|
125
130
|
let translation = null;
|
|
@@ -215,89 +220,92 @@ function runAsJob(_a) {
|
|
|
215
220
|
}
|
|
216
221
|
});
|
|
217
222
|
}
|
|
223
|
+
function reRunJob(_a) {
|
|
224
|
+
return __awaiter(this, arguments, void 0, function* ({ config, activeJob }) {
|
|
225
|
+
const storage = (0, storage_1.getStorage)();
|
|
226
|
+
if (activeJob && ![types_1.JobStatus.IN_PROGRESS, types_1.JobStatus.CREATED].includes(activeJob.status)) {
|
|
227
|
+
return;
|
|
228
|
+
}
|
|
229
|
+
if (activeJob.attempt && activeJob.attempt >= maxAttempts) {
|
|
230
|
+
yield storage.updateJob({ id: activeJob.id, status: types_1.JobStatus.FAILED });
|
|
231
|
+
return;
|
|
232
|
+
}
|
|
233
|
+
const { integrationId, crowdinId, type, title, payload } = activeJob;
|
|
234
|
+
const integration = config.projectIntegration;
|
|
235
|
+
let crowdinClient;
|
|
236
|
+
const integrationCredentials = yield storage.getIntegrationCredentials(integrationId);
|
|
237
|
+
const crowdinCredentials = yield storage.getCrowdinCredentials(crowdinId);
|
|
238
|
+
const integrationConfig = yield storage.getIntegrationConfig(integrationId);
|
|
239
|
+
if (!integrationCredentials || !crowdinCredentials) {
|
|
240
|
+
return;
|
|
241
|
+
}
|
|
242
|
+
const projectId = (0, token_1.getProjectId)(integrationCredentials.id);
|
|
243
|
+
const context = (0, context_1.crowdinContextInfoFromCredentials)({
|
|
244
|
+
projectId,
|
|
245
|
+
crowdinCredentials,
|
|
246
|
+
integrationCredentials,
|
|
247
|
+
});
|
|
248
|
+
(0, logger_1.setLogCrowdinContext)(context);
|
|
249
|
+
try {
|
|
250
|
+
const preparedCrowdinClient = yield (0, connection_1.prepareCrowdinClient)({
|
|
251
|
+
config,
|
|
252
|
+
credentials: crowdinCredentials,
|
|
253
|
+
autoRenew: true,
|
|
254
|
+
context,
|
|
255
|
+
});
|
|
256
|
+
crowdinClient = preparedCrowdinClient.client;
|
|
257
|
+
}
|
|
258
|
+
catch (e) {
|
|
259
|
+
(0, logger_1.logError)(e);
|
|
260
|
+
return;
|
|
261
|
+
}
|
|
262
|
+
const rootFolder = yield (0, defaults_1.getRootFolder)(config, integration, crowdinClient, projectId);
|
|
263
|
+
const credentials = yield (0, connection_1.prepareIntegrationCredentials)(config, integration, integrationCredentials);
|
|
264
|
+
const intConfig = (integrationConfig === null || integrationConfig === void 0 ? void 0 : integrationConfig.config)
|
|
265
|
+
? JSON.parse(integrationConfig.config)
|
|
266
|
+
: { schedule: '0', condition: '0' };
|
|
267
|
+
yield storage.updateJob({ attempt: +(activeJob.attempt || 0) + 1, id: activeJob.id });
|
|
268
|
+
if ([types_1.JobType.UPDATE_TO_CROWDIN, types_1.JobType.UPDATE_TO_INTEGRATION].includes(type)) {
|
|
269
|
+
yield (0, cron_1.runUpdateProviderJob)({
|
|
270
|
+
integrationId,
|
|
271
|
+
crowdinId,
|
|
272
|
+
type,
|
|
273
|
+
title,
|
|
274
|
+
payload: JSON.parse(payload),
|
|
275
|
+
jobType: types_1.JobClientType.RERUN,
|
|
276
|
+
projectId,
|
|
277
|
+
client: crowdinClient,
|
|
278
|
+
integration,
|
|
279
|
+
context,
|
|
280
|
+
credentials,
|
|
281
|
+
rootFolder,
|
|
282
|
+
settings: intConfig,
|
|
283
|
+
reRunJobId: activeJob.id,
|
|
284
|
+
});
|
|
285
|
+
}
|
|
286
|
+
else if ([types_1.JobType.CROWDIN_SYNC_SETTINGS_SAVE, types_1.JobType.INTEGRATION_SYNC_SETTINGS_SAVE].includes(type)) {
|
|
287
|
+
yield storage.updateJob({ status: types_1.JobStatus.CANCELED, id: activeJob.id });
|
|
288
|
+
}
|
|
289
|
+
});
|
|
290
|
+
}
|
|
218
291
|
function reRunInProgressJobs(config) {
|
|
219
292
|
return __awaiter(this, void 0, void 0, function* () {
|
|
220
|
-
const
|
|
221
|
-
const inProgressJobs = yield storage.getAllInProgressJobs();
|
|
293
|
+
const inProgressJobs = yield (0, storage_1.getStorage)().getAllInProgressJobs();
|
|
222
294
|
if (!(inProgressJobs === null || inProgressJobs === void 0 ? void 0 : inProgressJobs.length)) {
|
|
223
295
|
return;
|
|
224
296
|
}
|
|
225
297
|
for (let i = 0; i < inProgressJobs.length; i += RERUN_JOB_CONCURRENCY) {
|
|
226
298
|
const batch = inProgressJobs.slice(i, i + RERUN_JOB_CONCURRENCY);
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
}
|
|
231
|
-
if (activeJob.attempt && activeJob.attempt >= maxAttempts) {
|
|
232
|
-
yield storage.updateJob({ id: activeJob.id, status: types_1.JobStatus.FAILED });
|
|
233
|
-
return;
|
|
234
|
-
}
|
|
235
|
-
const { integrationId, crowdinId, type, title, payload } = activeJob;
|
|
236
|
-
const integration = config.projectIntegration;
|
|
237
|
-
let crowdinClient;
|
|
238
|
-
const integrationCredentials = yield (0, storage_1.getStorage)().getIntegrationCredentials(integrationId);
|
|
239
|
-
const crowdinCredentials = yield (0, storage_1.getStorage)().getCrowdinCredentials(crowdinId);
|
|
240
|
-
const integrationConfig = yield (0, storage_1.getStorage)().getIntegrationConfig(integrationId);
|
|
241
|
-
if (!integrationCredentials || !crowdinCredentials) {
|
|
242
|
-
return;
|
|
243
|
-
}
|
|
244
|
-
const projectId = (0, token_1.getProjectId)(integrationCredentials.id);
|
|
245
|
-
const context = {
|
|
246
|
-
jwtPayload: {
|
|
247
|
-
context: {
|
|
248
|
-
project_id: projectId,
|
|
249
|
-
organization_id: crowdinCredentials.organizationId,
|
|
250
|
-
organization_domain: crowdinCredentials.domain,
|
|
251
|
-
user_id: crowdinCredentials.userId,
|
|
252
|
-
},
|
|
253
|
-
},
|
|
254
|
-
};
|
|
299
|
+
yield Promise.all(
|
|
300
|
+
// re-runs start at app.listen, outside any request, so each job gets a scope of its own
|
|
301
|
+
batch.map((activeJob) => (0, logs_formatter_1.runWithContext)({}, () => __awaiter(this, void 0, void 0, function* () {
|
|
255
302
|
try {
|
|
256
|
-
|
|
257
|
-
config,
|
|
258
|
-
credentials: crowdinCredentials,
|
|
259
|
-
autoRenew: true,
|
|
260
|
-
context,
|
|
261
|
-
});
|
|
262
|
-
crowdinClient = preparedCrowdinClient.client;
|
|
303
|
+
yield reRunJob({ config, activeJob });
|
|
263
304
|
}
|
|
264
305
|
catch (e) {
|
|
265
306
|
(0, logger_1.logError)(e);
|
|
266
|
-
return;
|
|
267
307
|
}
|
|
268
|
-
|
|
269
|
-
const credentials = yield (0, connection_1.prepareIntegrationCredentials)(config, integration, integrationCredentials);
|
|
270
|
-
const intConfig = (integrationConfig === null || integrationConfig === void 0 ? void 0 : integrationConfig.config)
|
|
271
|
-
? JSON.parse(integrationConfig.config)
|
|
272
|
-
: { schedule: '0', condition: '0' };
|
|
273
|
-
yield storage.updateJob({ attempt: +(activeJob.attempt || 0) + 1, id: activeJob.id });
|
|
274
|
-
if ([types_1.JobType.UPDATE_TO_CROWDIN, types_1.JobType.UPDATE_TO_INTEGRATION].includes(type)) {
|
|
275
|
-
yield (0, cron_1.runUpdateProviderJob)({
|
|
276
|
-
integrationId,
|
|
277
|
-
crowdinId,
|
|
278
|
-
type,
|
|
279
|
-
title,
|
|
280
|
-
payload: JSON.parse(payload),
|
|
281
|
-
jobType: types_1.JobClientType.RERUN,
|
|
282
|
-
projectId,
|
|
283
|
-
client: crowdinClient,
|
|
284
|
-
integration,
|
|
285
|
-
context,
|
|
286
|
-
credentials,
|
|
287
|
-
rootFolder,
|
|
288
|
-
settings: intConfig,
|
|
289
|
-
reRunJobId: activeJob.id,
|
|
290
|
-
});
|
|
291
|
-
}
|
|
292
|
-
else if ([types_1.JobType.CROWDIN_SYNC_SETTINGS_SAVE, types_1.JobType.INTEGRATION_SYNC_SETTINGS_SAVE].includes(type)) {
|
|
293
|
-
yield storage.updateJob({ status: types_1.JobStatus.CANCELED, id: activeJob.id });
|
|
294
|
-
}
|
|
295
|
-
})));
|
|
296
|
-
for (const result of results) {
|
|
297
|
-
if (result.status === 'rejected') {
|
|
298
|
-
(0, logger_1.logError)(result.reason);
|
|
299
|
-
}
|
|
300
|
-
}
|
|
308
|
+
}))));
|
|
301
309
|
}
|
|
302
310
|
});
|
|
303
311
|
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import Crowdin from '@crowdin/crowdin-api-client';
|
|
2
|
+
import { Channel, ConsumeMessage } from 'amqplib';
|
|
2
3
|
import { Config, CrowdinContextInfo } from '../../../types';
|
|
3
4
|
import { AppSettings, IntegrationLogic, IntegrationSyncSettings, Provider, UpdateCrowdinWebhookPayloadsArgs } from '../types';
|
|
4
5
|
export declare const HookEvents: any;
|
|
@@ -35,6 +36,7 @@ export declare function prepareWebhookData({ config, integration, provider, webh
|
|
|
35
36
|
settings: AppSettings;
|
|
36
37
|
syncSettings: null;
|
|
37
38
|
preparedIntegrationCredentials: null;
|
|
39
|
+
logContext?: undefined;
|
|
38
40
|
} | {
|
|
39
41
|
projectId: number;
|
|
40
42
|
crowdinClient: {
|
|
@@ -45,6 +47,7 @@ export declare function prepareWebhookData({ config, integration, provider, webh
|
|
|
45
47
|
rootFolder: import("@crowdin/crowdin-api-client").SourceFilesModel.Directory | undefined;
|
|
46
48
|
settings: AppSettings;
|
|
47
49
|
syncSettings: IntegrationSyncSettings | null;
|
|
50
|
+
logContext: import("@crowdin/logs-formatter").Context;
|
|
48
51
|
}>;
|
|
49
52
|
export declare function updateCrowdinFromWebhookRequest(args: UpdateCrowdinWebhookPayloadsArgs): Promise<void>;
|
|
50
53
|
export declare function listenQueueMessage({ config, integration, queueName, queueUrl, }: {
|
|
@@ -53,3 +56,11 @@ export declare function listenQueueMessage({ config, integration, queueName, que
|
|
|
53
56
|
queueUrl: string;
|
|
54
57
|
queueName: string;
|
|
55
58
|
}): Promise<void>;
|
|
59
|
+
export declare function processMessages({ channel, messagesToAck, webhooksData, webhooksInfo, }: {
|
|
60
|
+
webhooksData: Promise<void>[];
|
|
61
|
+
webhooksInfo: {
|
|
62
|
+
[key: string]: any;
|
|
63
|
+
};
|
|
64
|
+
channel: Channel;
|
|
65
|
+
messagesToAck: ConsumeMessage[];
|
|
66
|
+
}): Promise<void>;
|