emailengine-app 2.69.0 → 2.71.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/deploy.yml +6 -3
- package/.github/workflows/release.yaml +2 -0
- package/.github/workflows/test.yml +73 -12
- package/.ncurc.js +3 -3
- package/CHANGELOG.md +37 -0
- package/Gruntfile.js +21 -23
- package/bin/emailengine.js +8 -1
- package/config/default.toml +5 -0
- package/config/test.toml +5 -0
- package/data/google-crawlers.json +1 -1
- package/getswagger.sh +44 -4
- package/gettext-extract.js +163 -0
- package/lib/account.js +104 -72
- package/lib/api-routes/account-routes.js +231 -71
- package/lib/api-routes/blocklist-routes.js +25 -18
- package/lib/api-routes/chat-routes.js +32 -14
- package/lib/api-routes/delivery-test-routes.js +30 -5
- package/lib/api-routes/export-routes.js +27 -2
- package/lib/api-routes/gateway-routes.js +63 -12
- package/lib/api-routes/license-routes.js +18 -4
- package/lib/api-routes/mailbox-routes.js +33 -7
- package/lib/api-routes/message-routes.js +291 -145
- package/lib/api-routes/oauth2-app-routes.js +90 -24
- package/lib/api-routes/outbox-routes.js +16 -4
- package/lib/api-routes/pubsub-routes.js +8 -4
- package/lib/api-routes/route-helpers.js +14 -1
- package/lib/api-routes/settings-routes.js +51 -25
- package/lib/api-routes/stats-routes.js +37 -3
- package/lib/api-routes/submit-routes.js +31 -42
- package/lib/api-routes/template-routes.js +54 -21
- package/lib/api-routes/token-routes.js +67 -67
- package/lib/api-routes/webhook-route-routes.js +37 -8
- package/lib/autodetect-imap-settings.js +0 -2
- package/lib/consts.js +5 -0
- package/lib/document-store.js +22 -1
- package/lib/email-client/base-client.js +31 -8
- package/lib/email-client/gmail-client.js +119 -112
- package/lib/email-client/imap/mailbox.js +2 -2
- package/lib/email-client/imap/subconnection.js +0 -1
- package/lib/email-client/imap/sync-operations.js +1 -1
- package/lib/email-client/imap-client.js +36 -17
- package/lib/email-client/notification-handler.js +3 -6
- package/lib/email-client/outlook-client.js +49 -62
- package/lib/export.js +49 -1
- package/lib/feature-flags.js +8 -2
- package/lib/gateway.js +4 -9
- package/lib/get-raw-email.js +5 -5
- package/lib/imapproxy/imap-core/lib/imap-connection.js +0 -1
- package/lib/license-beacon.js +367 -0
- package/lib/logger.js +35 -22
- package/lib/metrics-collector.js +0 -2
- package/lib/oauth2-apps.js +13 -4
- package/lib/outbox.js +24 -40
- package/lib/redis-operations.js +1 -1
- package/lib/routes-ui.js +2 -1
- package/lib/schemas.js +403 -83
- package/lib/sentry.js +139 -0
- package/lib/settings.js +9 -3
- package/lib/stream-encrypt.js +1 -1
- package/lib/templates.js +1 -1
- package/lib/tokens.js +5 -3
- package/lib/tools.js +28 -6
- package/lib/ui-routes/account-routes.js +7 -4
- package/lib/ui-routes/admin-config-routes.js +20 -6
- package/lib/ui-routes/document-store-routes.js +7 -1
- package/lib/ui-routes/oauth-config-routes.js +0 -2
- package/lib/ui-routes/route-helpers.js +0 -2
- package/lib/ui-routes/unsubscribe-routes.js +0 -2
- package/lib/webhooks.js +8 -4
- package/package.json +23 -19
- package/sbom.json +1 -1
- package/server.js +38 -31
- package/static/licenses.html +171 -391
- package/translations/de.mo +0 -0
- package/translations/de.po +154 -142
- package/translations/et.mo +0 -0
- package/translations/et.po +129 -131
- package/translations/fr.mo +0 -0
- package/translations/fr.po +133 -136
- package/translations/ja.mo +0 -0
- package/translations/ja.po +126 -129
- package/translations/messages.pot +107 -107
- package/translations/nl.mo +0 -0
- package/translations/nl.po +128 -130
- package/translations/pl.mo +0 -0
- package/translations/pl.po +125 -128
- package/update-info.sh +19 -1
- package/views/config/logging.hbs +48 -0
- package/views/dashboard.hbs +22 -0
- package/workers/api.js +33 -37
- package/workers/documents.js +2 -22
- package/workers/export.js +73 -92
- package/workers/imap-proxy.js +3 -23
- package/workers/imap.js +2 -22
- package/workers/smtp.js +2 -22
- package/workers/submit.js +6 -24
- package/workers/webhooks.js +2 -22
package/lib/sentry.js
ADDED
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const packageData = require('../package.json');
|
|
4
|
+
const logger = require('./logger');
|
|
5
|
+
const { readEnvValue } = require('./tools');
|
|
6
|
+
const { COMMUNITY_SENTRY_DSN } = require('./consts');
|
|
7
|
+
|
|
8
|
+
const SENTRY_SETTINGS_CHECK_INTERVAL = 60 * 1000;
|
|
9
|
+
|
|
10
|
+
let Sentry;
|
|
11
|
+
let workerName;
|
|
12
|
+
let activeDsn = false;
|
|
13
|
+
|
|
14
|
+
function startSentry(dsn) {
|
|
15
|
+
// require lazily, the SDK loads several hundred modules in every worker thread,
|
|
16
|
+
// so only pay that cost when error tracking is actually enabled
|
|
17
|
+
if (!Sentry) {
|
|
18
|
+
Sentry = require('@sentry/node');
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
Sentry.init({
|
|
22
|
+
dsn,
|
|
23
|
+
release: packageData.version,
|
|
24
|
+
// Error capture only: skip the OpenTelemetry setup and the default
|
|
25
|
+
// integrations that patch http/fetch/console on hot paths. Sentry's own
|
|
26
|
+
// uncaughtException/unhandledRejection integrations are left out on
|
|
27
|
+
// purpose - they do not run in worker threads and do not guarantee a
|
|
28
|
+
// process exit, so lib/logger.js owns reporting and exiting instead.
|
|
29
|
+
skipOpenTelemetrySetup: true,
|
|
30
|
+
defaultIntegrations: false,
|
|
31
|
+
integrations: [
|
|
32
|
+
Sentry.eventFiltersIntegration(),
|
|
33
|
+
Sentry.functionToStringIntegration(),
|
|
34
|
+
Sentry.linkedErrorsIntegration(),
|
|
35
|
+
Sentry.contextLinesIntegration(),
|
|
36
|
+
Sentry.nodeContextIntegration(),
|
|
37
|
+
Sentry.modulesIntegration()
|
|
38
|
+
],
|
|
39
|
+
initialScope: {
|
|
40
|
+
tags: { worker: workerName }
|
|
41
|
+
}
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
logger.notifyError = (err, opts) => {
|
|
45
|
+
let captureContext = {};
|
|
46
|
+
if (opts?.user) {
|
|
47
|
+
captureContext.user = { id: `${opts.user}` };
|
|
48
|
+
}
|
|
49
|
+
if (opts?.meta && Object.keys(opts.meta).length) {
|
|
50
|
+
captureContext.contexts = { ee: opts.meta };
|
|
51
|
+
}
|
|
52
|
+
Sentry.captureException(err, captureContext);
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
// the global exception handlers in lib/logger.js wait for this before exiting
|
|
56
|
+
logger.flushNotifications = () => Sentry.flush(2000);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
// Tag all events with the installation identity, so reports from different
|
|
60
|
+
// EmailEngine instances sharing the same DSN can be told apart. Read once per
|
|
61
|
+
// SDK start - a license swapped at runtime is reflected after the next restart.
|
|
62
|
+
async function applyIdentityTags() {
|
|
63
|
+
const settings = require('./settings');
|
|
64
|
+
let { serviceId, tract } = await settings.getMulti('serviceId', 'tract');
|
|
65
|
+
let tags = {};
|
|
66
|
+
if (serviceId) {
|
|
67
|
+
tags.instance = serviceId;
|
|
68
|
+
}
|
|
69
|
+
if (tract?.key) {
|
|
70
|
+
tags.license = tract.key;
|
|
71
|
+
}
|
|
72
|
+
Sentry.getGlobalScope().setTags(tags);
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
async function applySentryState(dsn) {
|
|
76
|
+
dsn = dsn || false;
|
|
77
|
+
if (dsn === activeDsn) {
|
|
78
|
+
return;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
if (activeDsn) {
|
|
82
|
+
delete logger.notifyError;
|
|
83
|
+
delete logger.flushNotifications;
|
|
84
|
+
await Sentry.close(2000);
|
|
85
|
+
logger.info({ msg: 'Disabled Sentry error reporting', worker: workerName });
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
if (dsn) {
|
|
89
|
+
startSentry(dsn);
|
|
90
|
+
logger.info({ msg: 'Enabled Sentry error reporting', worker: workerName });
|
|
91
|
+
|
|
92
|
+
// identity lookup failure must not break error reporting, events just lack the tags
|
|
93
|
+
applyIdentityTags().catch(err => {
|
|
94
|
+
logger.error({ msg: 'Failed to apply Sentry identity tags', worker: workerName, err });
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
activeDsn = dsn;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
async function checkSentrySettings() {
|
|
102
|
+
const settings = require('./settings');
|
|
103
|
+
let { sentryEnabled, sentryDsn } = await settings.getMulti('sentryEnabled', 'sentryDsn');
|
|
104
|
+
await applySentryState(sentryEnabled ? sentryDsn || COMMUNITY_SENTRY_DSN : false);
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
// Initialize Sentry error tracking. If the SENTRY_DSN environment variable is set,
|
|
108
|
+
// it pins the configuration and runtime settings are ignored. Otherwise the
|
|
109
|
+
// `sentryEnabled` and `sentryDsn` settings are applied at runtime and re-checked
|
|
110
|
+
// periodically, so error reporting can be toggled from the admin UI without a
|
|
111
|
+
// restart. While disabled, logger.notifyError stays undefined and the global
|
|
112
|
+
// exception handlers in lib/logger.js exit without waiting for a delivery flush.
|
|
113
|
+
function initSentry(worker) {
|
|
114
|
+
workerName = worker;
|
|
115
|
+
|
|
116
|
+
let envDsn = readEnvValue('SENTRY_DSN');
|
|
117
|
+
if (envDsn) {
|
|
118
|
+
applySentryState(envDsn).catch(err => {
|
|
119
|
+
logger.error({ msg: 'Failed to initialize Sentry', worker: workerName, err });
|
|
120
|
+
});
|
|
121
|
+
return;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
// Settings changes are detected by polling instead of the {cmd: 'settings'}
|
|
125
|
+
// broadcast because that broadcast does not reach all the threads that
|
|
126
|
+
// initialize Sentry (smtp, imap-proxy, documents, and the main thread).
|
|
127
|
+
let checkSettings = async () => {
|
|
128
|
+
try {
|
|
129
|
+
await checkSentrySettings();
|
|
130
|
+
} catch (err) {
|
|
131
|
+
logger.error({ msg: 'Failed to apply Sentry settings', worker: workerName, err });
|
|
132
|
+
}
|
|
133
|
+
setTimeout(checkSettings, SENTRY_SETTINGS_CHECK_INTERVAL).unref();
|
|
134
|
+
};
|
|
135
|
+
|
|
136
|
+
setImmediate(checkSettings);
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
module.exports = { initSentry };
|
package/lib/settings.js
CHANGED
|
@@ -13,9 +13,12 @@ config.service = config.service || {};
|
|
|
13
13
|
const ENCRYPTED_KEYS = [
|
|
14
14
|
'gmailClientSecret',
|
|
15
15
|
'outlookClientSecret',
|
|
16
|
+
'mailRuClientSecret',
|
|
16
17
|
'cookiePassword',
|
|
17
18
|
'smtpServerPassword',
|
|
19
|
+
'imapProxyServerPassword',
|
|
18
20
|
'serviceSecret',
|
|
21
|
+
'serviceKey',
|
|
19
22
|
'gmailServiceKey',
|
|
20
23
|
'gmailServiceExternalAccount',
|
|
21
24
|
'documentStorePassword',
|
|
@@ -100,6 +103,9 @@ module.exports = {
|
|
|
100
103
|
formatSettingValue(key, value) {
|
|
101
104
|
switch (key) {
|
|
102
105
|
case 'serviceUrl': {
|
|
106
|
+
if (!value) {
|
|
107
|
+
return value;
|
|
108
|
+
}
|
|
103
109
|
let urlObj = new URL(value);
|
|
104
110
|
return urlObj.origin;
|
|
105
111
|
}
|
|
@@ -155,9 +161,9 @@ module.exports = {
|
|
|
155
161
|
}
|
|
156
162
|
}
|
|
157
163
|
|
|
158
|
-
if (
|
|
159
|
-
// make sure `notifyText` is enabled
|
|
160
|
-
|
|
164
|
+
if (['generateEmailSummary', 'openAiGenerateEmbeddings'].includes(key) && formattedValue) {
|
|
165
|
+
// AI processing needs access to message text content, so make sure `notifyText` is enabled as well
|
|
166
|
+
await module.exports.set('notifyText', true);
|
|
161
167
|
}
|
|
162
168
|
|
|
163
169
|
if (/^openAi/.test(key) || key === 'generateEmailSummary') {
|
package/lib/stream-encrypt.js
CHANGED
|
@@ -162,7 +162,7 @@ class DecryptStream extends Transform {
|
|
|
162
162
|
decrypted = Buffer.concat([decipher.update(encryptedData), decipher.final()]);
|
|
163
163
|
} catch (err) {
|
|
164
164
|
if (err.message.includes('auth')) {
|
|
165
|
-
throw new Error('Decryption failed: invalid secret or corrupted data');
|
|
165
|
+
throw new Error('Decryption failed: invalid secret or corrupted data', { cause: err });
|
|
166
166
|
}
|
|
167
167
|
throw err;
|
|
168
168
|
}
|
package/lib/templates.js
CHANGED
|
@@ -53,7 +53,7 @@ class TemplateHandler {
|
|
|
53
53
|
let templateMeta = msgpack.decode(entry);
|
|
54
54
|
response.templates.push(templateMeta);
|
|
55
55
|
} catch (err) {
|
|
56
|
-
logger.error({ msg: 'Failed to process template', entry: entry.toString('base64') });
|
|
56
|
+
logger.error({ msg: 'Failed to process template', entry: entry && entry.toString('base64'), err });
|
|
57
57
|
continue;
|
|
58
58
|
}
|
|
59
59
|
}
|
package/lib/tokens.js
CHANGED
|
@@ -325,16 +325,18 @@ module.exports = {
|
|
|
325
325
|
let lastEntry = false;
|
|
326
326
|
for (let i = 0; i < detailList.length; i++) {
|
|
327
327
|
let entry = detailList[i];
|
|
328
|
+
// Each token occupies two consecutive multi results (data + access info)
|
|
329
|
+
let tokenHash = list[Math.floor(i / 2)];
|
|
328
330
|
if (i % 2 === 0) {
|
|
329
331
|
lastEntry = false;
|
|
330
332
|
if (entry[1]) {
|
|
331
333
|
try {
|
|
332
334
|
let tokenData = msgpack.decode(entry[1]);
|
|
333
335
|
tokenData.created = new Date(tokenData.created);
|
|
334
|
-
lastEntry = Object.assign({ id:
|
|
336
|
+
lastEntry = Object.assign({ id: tokenHash }, tokenData);
|
|
335
337
|
response.tokens.push(lastEntry);
|
|
336
338
|
} catch (err) {
|
|
337
|
-
logger.error({ msg: 'Failed to process token data', hash:
|
|
339
|
+
logger.error({ msg: 'Failed to process token data', hash: tokenHash, err });
|
|
338
340
|
}
|
|
339
341
|
}
|
|
340
342
|
} else if (lastEntry && entry[1]) {
|
|
@@ -343,7 +345,7 @@ module.exports = {
|
|
|
343
345
|
accessData.time = accessData.time ? new Date(accessData.time) : null;
|
|
344
346
|
lastEntry.access = accessData;
|
|
345
347
|
} catch (err) {
|
|
346
|
-
logger.error({ msg: 'Failed to process token data', hash:
|
|
348
|
+
logger.error({ msg: 'Failed to process token data', hash: tokenHash, err });
|
|
347
349
|
}
|
|
348
350
|
}
|
|
349
351
|
}
|
package/lib/tools.js
CHANGED
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
/* eslint no-bitwise: 0 */
|
|
2
2
|
|
|
3
|
-
// NB! This file is processed by gettext parser and can not use newer syntax like ?.
|
|
4
|
-
|
|
5
3
|
'use strict';
|
|
6
4
|
|
|
7
5
|
const msgpack = require('msgpack5')();
|
|
@@ -59,7 +57,26 @@ const AGENT_OPTS = {
|
|
|
59
57
|
const RETRY_OPTS = {
|
|
60
58
|
maxRetries: URL_FETCH_RETRY_MAX,
|
|
61
59
|
methods: ['GET', 'PUT', 'HEAD', 'OPTIONS', 'DELETE', 'POST'],
|
|
62
|
-
statusCodes: [429] // do not retry 5xx errors
|
|
60
|
+
statusCodes: [429], // do not retry 5xx errors
|
|
61
|
+
// undici does not retry transient DNS failures (EAI_AGAIN) or socket connect
|
|
62
|
+
// timeouts (ETIMEDOUT) by default, so a name-resolution blip bubbles up as a hard
|
|
63
|
+
// `TypeError: fetch failed`. Add them to the retry list. Passing errorCodes REPLACES
|
|
64
|
+
// undici's defaults, so its default codes are re-listed here.
|
|
65
|
+
errorCodes: [
|
|
66
|
+
// undici defaults
|
|
67
|
+
'ECONNRESET',
|
|
68
|
+
'ECONNREFUSED',
|
|
69
|
+
'ENOTFOUND',
|
|
70
|
+
'ENETDOWN',
|
|
71
|
+
'ENETUNREACH',
|
|
72
|
+
'EHOSTDOWN',
|
|
73
|
+
'EHOSTUNREACH',
|
|
74
|
+
'EPIPE',
|
|
75
|
+
'UND_ERR_SOCKET',
|
|
76
|
+
// additions: transient DNS / socket connect-timeout failures
|
|
77
|
+
'EAI_AGAIN',
|
|
78
|
+
'ETIMEDOUT'
|
|
79
|
+
]
|
|
63
80
|
};
|
|
64
81
|
|
|
65
82
|
// Shared mutable object -- consumers import the object reference and access
|
|
@@ -1517,9 +1534,9 @@ MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEV3QUiYsp13nD9suD1/ZkEXnuMoSg
|
|
|
1517
1534
|
.hget(`${REDIS_PREFIX}bull:${queue}:meta`, 'paused')
|
|
1518
1535
|
.exec();
|
|
1519
1536
|
if (resActive[0] || resDelayed[0] || resWaiting[0] || resPaused[0] || resMeta[0]) {
|
|
1520
|
-
// counting failed
|
|
1537
|
+
// counting failed, skip this queue but keep the rest of the stats response
|
|
1521
1538
|
logger.error({ msg: 'Failed to count queue length', queue, active: resActive, delayed: resDelayed, waiting: resWaiting });
|
|
1522
|
-
|
|
1539
|
+
continue;
|
|
1523
1540
|
}
|
|
1524
1541
|
queues[queue] = {
|
|
1525
1542
|
active: Number(resActive[1]) || 0,
|
|
@@ -2200,6 +2217,7 @@ vWuuhT9ely8AUX2F
|
|
|
2200
2217
|
mF3GOI+Ev7mJODtG
|
|
2201
2218
|
nQNwPlZ+tyx24XVo
|
|
2202
2219
|
olObiSmdzVaMp7lH
|
|
2220
|
+
SruXHv6plKL9YuAW
|
|
2203
2221
|
cyp1GKV4Cl+eC8G/
|
|
2204
2222
|
Q1y/TzbtUQCqnotR
|
|
2205
2223
|
59Q2qOkuyTLzQDhd
|
|
@@ -2223,9 +2241,13 @@ EsbWnuADOq9qe/EZ
|
|
|
2223
2241
|
YfUSxQtq1+kpwW/W
|
|
2224
2242
|
QodzxvoJ6NPGhCgW
|
|
2225
2243
|
5/VK+O950efBio0t
|
|
2226
|
-
RA/lrkwxcTX80nxX
|
|
2227
2244
|
b85BMAFRHn10uX8y
|
|
2228
2245
|
vV2RgfFH2YFTYsly
|
|
2246
|
+
zcmIO3obSzUVHGtF
|
|
2247
|
+
tzsA68uFVAffHmec
|
|
2248
|
+
DkCU3OpaSa+rIVfU
|
|
2249
|
+
dcDjqpxVxivT46Rl
|
|
2250
|
+
QSgNcpNukC9CBNfe
|
|
2229
2251
|
`
|
|
2230
2252
|
.split(/\r?\n/)
|
|
2231
2253
|
.map(l => l.trim())
|
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
|
-
// NB! This file is processed by the gettext parser (npm run gettext) and can not use newer syntax like ?.
|
|
4
|
-
|
|
5
3
|
// Admin UI routes for account management: /admin/accounts (listing), /admin/accounts/new and
|
|
6
4
|
// /accounts/new* (the add-account wizard incl. IMAP autoconfig/test/server steps), and
|
|
7
5
|
// /admin/accounts/{account}* (view, edit, delete, reconnect, sync, logs, browse). Extracted
|
|
@@ -33,7 +31,7 @@ const {
|
|
|
33
31
|
readEnvValue,
|
|
34
32
|
failAction
|
|
35
33
|
} = require('../tools');
|
|
36
|
-
const { settingsSchema, accountIdSchema, defaultAccountTypeSchema } = require('../schemas');
|
|
34
|
+
const { settingsSchema, accountIdSchema, defaultAccountTypeSchema, ACCOUNT_DISPLAY_STATES } = require('../schemas');
|
|
37
35
|
const { formatAccountData, cachedTemplates } = require('./route-helpers');
|
|
38
36
|
|
|
39
37
|
const { REDIS_PREFIX, DEFAULT_PAGE_SIZE, NONCE_BYTES, MAX_FORM_TTL, DEFAULT_MAX_LOG_LINES } = consts;
|
|
@@ -179,6 +177,11 @@ function init(args) {
|
|
|
179
177
|
label: 'Disconnected'
|
|
180
178
|
},
|
|
181
179
|
|
|
180
|
+
{
|
|
181
|
+
state: 'paused',
|
|
182
|
+
label: 'Paused'
|
|
183
|
+
},
|
|
184
|
+
|
|
182
185
|
{
|
|
183
186
|
state: 'authenticationError',
|
|
184
187
|
label: 'Authentication failed'
|
|
@@ -252,7 +255,7 @@ function init(args) {
|
|
|
252
255
|
state: Joi.string()
|
|
253
256
|
.trim()
|
|
254
257
|
.empty('')
|
|
255
|
-
.valid(
|
|
258
|
+
.valid(...ACCOUNT_DISPLAY_STATES)
|
|
256
259
|
.example('connected')
|
|
257
260
|
.description('Filter accounts by state')
|
|
258
261
|
.label('AccountState')
|
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
|
-
// NB! This file is processed by the gettext parser (npm run gettext) and can not use newer syntax like ?.
|
|
4
|
-
|
|
5
3
|
// Admin UI routes for the remaining /admin/config/* pages: webhooks, service URL/branding,
|
|
6
4
|
// AI (OpenAI) settings, logging, and license. Extracted verbatim from lib/routes-ui.js.
|
|
7
5
|
// The notificationTypes / configWebhooksSchema / configLoggingSchema consts, the
|
|
@@ -25,6 +23,7 @@ const timezonesList = require('timezones-list').default;
|
|
|
25
23
|
const { redis, submitQueue, notifyQueue, documentsQueue } = require('../db');
|
|
26
24
|
const { getByteSize, formatByteSize, getDuration, failAction, hasEnvValue, readEnvValue, httpAgent } = require('../tools');
|
|
27
25
|
const { llmPreProcess } = require('../llm-pre-process');
|
|
26
|
+
const { documentStoreFeatureEnabled } = require('../document-store');
|
|
28
27
|
const { locales } = require('../translations');
|
|
29
28
|
const { settingsSchema } = require('../schemas');
|
|
30
29
|
const { getOpenAiModels, OPEN_AI_MODELS, getExampleDocumentsPayloads } = require('./route-helpers');
|
|
@@ -131,7 +130,9 @@ for (let type of notificationTypes) {
|
|
|
131
130
|
|
|
132
131
|
const configLoggingSchema = {
|
|
133
132
|
all: Joi.boolean().truthy('Y', 'true', '1', 'on').falsy('N', 'false', 0, '').default(false).description('Enable logs for all accounts'),
|
|
134
|
-
maxLogLines: Joi.number().integer().empty('').min(0).max(10000000).default(DEFAULT_MAX_LOG_LINES)
|
|
133
|
+
maxLogLines: Joi.number().integer().empty('').min(0).max(10000000).default(DEFAULT_MAX_LOG_LINES),
|
|
134
|
+
sentryEnabled: settingsSchema.sentryEnabled.default(false),
|
|
135
|
+
sentryDsn: settingsSchema.sentryDsn.default('')
|
|
135
136
|
};
|
|
136
137
|
|
|
137
138
|
async function getOpenAiError(gt) {
|
|
@@ -217,7 +218,7 @@ function init(args) {
|
|
|
217
218
|
values,
|
|
218
219
|
|
|
219
220
|
webhookErrorFlag: await settings.get('webhookErrorFlag'),
|
|
220
|
-
documentStoreEnabled: (await settings.get('documentStoreEnabled')) || false
|
|
221
|
+
documentStoreEnabled: (documentStoreFeatureEnabled && (await settings.get('documentStoreEnabled'))) || false
|
|
221
222
|
},
|
|
222
223
|
{
|
|
223
224
|
layout: 'app'
|
|
@@ -307,7 +308,7 @@ function init(args) {
|
|
|
307
308
|
),
|
|
308
309
|
|
|
309
310
|
webhookErrorFlag: await settings.get('webhookErrorFlag'),
|
|
310
|
-
documentStoreEnabled: (await settings.get('documentStoreEnabled')) || false
|
|
311
|
+
documentStoreEnabled: (documentStoreFeatureEnabled && (await settings.get('documentStoreEnabled'))) || false
|
|
311
312
|
},
|
|
312
313
|
{
|
|
313
314
|
layout: 'app'
|
|
@@ -356,7 +357,7 @@ function init(args) {
|
|
|
356
357
|
errors,
|
|
357
358
|
|
|
358
359
|
webhookErrorFlag: await settings.get('webhookErrorFlag'),
|
|
359
|
-
documentStoreEnabled: (await settings.get('documentStoreEnabled')) || false
|
|
360
|
+
documentStoreEnabled: (documentStoreFeatureEnabled && (await settings.get('documentStoreEnabled'))) || false
|
|
360
361
|
},
|
|
361
362
|
{
|
|
362
363
|
layout: 'app'
|
|
@@ -1120,6 +1121,10 @@ return true;`
|
|
|
1120
1121
|
|
|
1121
1122
|
values.accounts = [].concat(values.accounts || []).join('\n');
|
|
1122
1123
|
|
|
1124
|
+
let sentryValues = await settings.getMulti('sentryEnabled', 'sentryDsn');
|
|
1125
|
+
values.sentryEnabled = !!sentryValues.sentryEnabled;
|
|
1126
|
+
values.sentryDsn = sentryValues.sentryDsn || '';
|
|
1127
|
+
|
|
1123
1128
|
return h.view(
|
|
1124
1129
|
'config/logging',
|
|
1125
1130
|
{
|
|
@@ -1127,6 +1132,8 @@ return true;`
|
|
|
1127
1132
|
menuConfig: true,
|
|
1128
1133
|
menuConfigLogging: true,
|
|
1129
1134
|
|
|
1135
|
+
sentryEnvManaged: !!readEnvValue('SENTRY_DSN'),
|
|
1136
|
+
|
|
1130
1137
|
values
|
|
1131
1138
|
},
|
|
1132
1139
|
{
|
|
@@ -1148,6 +1155,13 @@ return true;`
|
|
|
1148
1155
|
}
|
|
1149
1156
|
};
|
|
1150
1157
|
|
|
1158
|
+
if (!readEnvValue('SENTRY_DSN')) {
|
|
1159
|
+
// the form renders these fields as disabled when the DSN is pinned by the
|
|
1160
|
+
// environment, so do not overwrite the stored values in that case
|
|
1161
|
+
data.sentryEnabled = !!request.payload.sentryEnabled;
|
|
1162
|
+
data.sentryDsn = request.payload.sentryDsn || '';
|
|
1163
|
+
}
|
|
1164
|
+
|
|
1151
1165
|
for (let key of Object.keys(data)) {
|
|
1152
1166
|
await settings.set(key, data[key]);
|
|
1153
1167
|
}
|
|
@@ -16,7 +16,7 @@ const { REDIS_PREFIX } = require('../consts');
|
|
|
16
16
|
const { failAction } = require('../tools');
|
|
17
17
|
const { settingsSchema } = require('../schemas');
|
|
18
18
|
const { defaultMappings } = require('../es');
|
|
19
|
-
const { getESClient } = require('../document-store');
|
|
19
|
+
const { getESClient, documentStoreFeatureEnabled } = require('../document-store');
|
|
20
20
|
const { getOpenAiModels, OPEN_AI_MODELS, getExampleDocumentsPayloads } = require('./route-helpers');
|
|
21
21
|
|
|
22
22
|
const FIELD_TYPES = [
|
|
@@ -95,6 +95,12 @@ const configDocumentStoreSchema = {
|
|
|
95
95
|
function init(args) {
|
|
96
96
|
const { server } = args;
|
|
97
97
|
|
|
98
|
+
// Deprecated Document Store feature: when the gate is off, register no routes so that
|
|
99
|
+
// every /admin/config/document-store* page behaves like a regular 404.
|
|
100
|
+
if (!documentStoreFeatureEnabled) {
|
|
101
|
+
return;
|
|
102
|
+
}
|
|
103
|
+
|
|
98
104
|
server.route({
|
|
99
105
|
method: 'GET',
|
|
100
106
|
path: '/admin/config/document-store',
|
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
|
-
// NB! This file is processed by the gettext parser (npm run gettext) and can not use newer syntax like ?.
|
|
4
|
-
|
|
5
3
|
// Admin UI routes for OAuth2 application config (/admin/config/oauth*): listing apps,
|
|
6
4
|
// per-app view/edit/delete, creating apps, adding accounts, provider subscriptions, and
|
|
7
5
|
// app verification. Extracted verbatim from lib/routes-ui.js. AZURE_CLOUDS and the
|
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
|
-
// NB! This file is processed by the gettext parser (npm run gettext) and can not use newer syntax like ?.
|
|
4
|
-
|
|
5
3
|
// Shared helpers used by more than one extracted UI route module - and still by
|
|
6
4
|
// lib/routes-ui.js for the route groups not yet extracted. Lifting these here lets each
|
|
7
5
|
// consumer import the single canonical copy instead of the monolith, so a route group can
|
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
|
-
// NB! This file is processed by the gettext parser (npm run gettext) and can not use newer syntax like ?.
|
|
4
|
-
|
|
5
3
|
// Public (unauthenticated) subscription-management routes. These render the unsubscribe
|
|
6
4
|
// landing page reached from the List-Unsubscribe link in outgoing messages and process
|
|
7
5
|
// the subscribe/unsubscribe form submission. Extracted verbatim from lib/routes-ui.js.
|
package/lib/webhooks.js
CHANGED
|
@@ -70,12 +70,12 @@ class WebhooksHandler {
|
|
|
70
70
|
|
|
71
71
|
try {
|
|
72
72
|
let webhookMeta = msgpack.decode(entry);
|
|
73
|
-
if (webhookErrorFlag && typeof webhookErrorFlag === 'object' && !Object.keys(webhookErrorFlag)) {
|
|
73
|
+
if (webhookErrorFlag && typeof webhookErrorFlag === 'object' && !Object.keys(webhookErrorFlag).length) {
|
|
74
74
|
webhookErrorFlag = null;
|
|
75
75
|
}
|
|
76
76
|
response.webhooks.push(Object.assign(webhookMeta, { tcount, webhookErrorFlag }));
|
|
77
77
|
} catch (err) {
|
|
78
|
-
logger.error({ msg: 'Failed to process webhook', entry: entry.toString('base64') });
|
|
78
|
+
logger.error({ msg: 'Failed to process webhook', entry: entry && entry.toString('base64'), err });
|
|
79
79
|
continue;
|
|
80
80
|
}
|
|
81
81
|
}
|
|
@@ -398,7 +398,6 @@ class WebhooksHandler {
|
|
|
398
398
|
v = Number(v) || 0;
|
|
399
399
|
if (v !== this.handlerCacheV) {
|
|
400
400
|
// changes detected
|
|
401
|
-
v = this.handlerCacheV;
|
|
402
401
|
|
|
403
402
|
let webhookIds = await this.redis.smembers(this.getWebhooksIndexKey());
|
|
404
403
|
webhookIds = [].concat(webhookIds || []).sort((a, b) => -a.localeCompare(b));
|
|
@@ -418,7 +417,8 @@ class WebhooksHandler {
|
|
|
418
417
|
this.handlerCache.push(handler);
|
|
419
418
|
} else {
|
|
420
419
|
// compare existing
|
|
421
|
-
|
|
420
|
+
// the per-route counter is stored as a string, existing.v is a number (see get())
|
|
421
|
+
let webhookV = Number(await this.redis.hget(this.getWebhooksContentKey(), `${webhookId}:v`)) || 0;
|
|
422
422
|
if (existing.v !== webhookV) {
|
|
423
423
|
// update
|
|
424
424
|
for (let i = this.handlerCache.length - 1; i >= 0; i--) {
|
|
@@ -430,6 +430,10 @@ class WebhooksHandler {
|
|
|
430
430
|
}
|
|
431
431
|
}
|
|
432
432
|
}
|
|
433
|
+
|
|
434
|
+
// mark the cache as current only after a successful refresh, so a failure above
|
|
435
|
+
// leaves the cache stale and the next call retries
|
|
436
|
+
this.handlerCacheV = v;
|
|
433
437
|
}
|
|
434
438
|
|
|
435
439
|
return this.handlerCache;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "emailengine-app",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.71.0",
|
|
4
4
|
"private": false,
|
|
5
5
|
"productTitle": "EmailEngine",
|
|
6
6
|
"description": "Email Sync Engine",
|
|
@@ -11,13 +11,15 @@
|
|
|
11
11
|
"single": "EE_OPENAPI_VERBOSE=true EENGINE_LOG_RAW=true EENGINE_SECRET=your-encryption-key EENGINE_WORKERS=1 node --inspect server --dbs.redis='redis://127.0.0.1:6379/6' --api.port=7003 --api.host=0.0.0.0 | tee $HOME/ee.log.single.txt | pino-pretty",
|
|
12
12
|
"gmail": "EE_OPENAPI_VERBOSE=true EENGINE_LOG_RAW=true EENGINE_SECRET=your-encryption-key EENGINE_WORKERS=2 EENGINE_CORS_ORIGIN='*' node --inspect server --dbs.redis='redis://127.0.0.1:6379/11' --api.port=7003 --api.host=0.0.0.0 | tee $HOME/ee.log.gmail.txt | pino-pretty",
|
|
13
13
|
"test": "NODE_ENV=test grunt",
|
|
14
|
-
"
|
|
14
|
+
"test:unit": "NODE_ENV=test grunt test-unit",
|
|
15
|
+
"test:integration": "NODE_ENV=test grunt test-integration",
|
|
16
|
+
"lint": "npx eslint 'lib/**/*.js' 'workers/**/*.js' 'test/**/*.js' server.js Gruntfile.js",
|
|
15
17
|
"swagger": "./getswagger.sh",
|
|
16
18
|
"build-source": "rm -rf node_modules && npm install && rm -rf node_modules && npm ci --omit=dev && rm -rf node_modules/ace-builds node_modules/@postalsys/ee-client && ./update-info.sh",
|
|
17
19
|
"build-dist": "pkg --compress Brotli package.json && npm install && node winconf.js",
|
|
18
20
|
"build-dist-fast": "pkg --debug package.json && npm install && node winconf.js",
|
|
19
21
|
"licenses": "license-checker --excludePackages 'emailengine-app' --json | node list-generate.js > static/licenses.html",
|
|
20
|
-
"gettext": "find ./views -name \"*.hbs\" -print0 | xargs -0 xgettext-template -L Handlebars -o translations/messages.pot --force-po &&
|
|
22
|
+
"gettext": "find ./views -name \"*.hbs\" -print0 | xargs -0 xgettext-template -L Handlebars -o translations/messages.pot --force-po && node gettext-extract.js",
|
|
21
23
|
"prepare-docker": "echo \"EE_DOCKER_LEGACY=$EE_DOCKER_LEGACY\" >> system.env && cat system.env",
|
|
22
24
|
"update": "rm -rf node_modules package-lock.json && ncu -u && npm install && ./copy-static-files.sh && npm run licenses && npm run gettext",
|
|
23
25
|
"test-gmail-api": "node lib/email-client/gmail-client.js --dbs.redis=redis://127.0.0.1/11",
|
|
@@ -43,9 +45,8 @@
|
|
|
43
45
|
},
|
|
44
46
|
"homepage": "https://emailengine.app/",
|
|
45
47
|
"dependencies": {
|
|
46
|
-
"@
|
|
47
|
-
"@bull-board/
|
|
48
|
-
"@bull-board/hapi": "7.2.1",
|
|
48
|
+
"@bull-board/api": "8.0.0",
|
|
49
|
+
"@bull-board/hapi": "8.0.0",
|
|
49
50
|
"@elastic/elasticsearch": "8.15.3",
|
|
50
51
|
"@hapi/accept": "6.0.3",
|
|
51
52
|
"@hapi/bell": "13.1.0",
|
|
@@ -57,20 +58,21 @@
|
|
|
57
58
|
"@hapi/vision": "7.0.3",
|
|
58
59
|
"@phc/pbkdf2": "1.1.14",
|
|
59
60
|
"@postalsys/bounce-classifier": "3.0.0",
|
|
60
|
-
"@postalsys/certs": "1.0.
|
|
61
|
+
"@postalsys/certs": "1.0.15",
|
|
61
62
|
"@postalsys/ee-client": "1.3.0",
|
|
62
|
-
"@postalsys/email-ai-tools": "1.13.
|
|
63
|
-
"@postalsys/email-text-tools": "2.4.
|
|
63
|
+
"@postalsys/email-ai-tools": "1.13.6",
|
|
64
|
+
"@postalsys/email-text-tools": "2.4.7",
|
|
64
65
|
"@postalsys/gettext": "4.1.1",
|
|
65
66
|
"@postalsys/joi-messages": "1.0.5",
|
|
66
67
|
"@postalsys/templates": "2.0.1",
|
|
68
|
+
"@sentry/node": "10.58.0",
|
|
67
69
|
"@simplewebauthn/browser": "13.3.0",
|
|
68
70
|
"@simplewebauthn/server": "13.3.1",
|
|
69
71
|
"@zone-eu/mailsplit": "5.4.12",
|
|
70
72
|
"@zone-eu/wild-config": "1.7.5",
|
|
71
73
|
"ace-builds": "1.44.0",
|
|
72
74
|
"base32.js": "0.1.0",
|
|
73
|
-
"bullmq": "5.78.
|
|
75
|
+
"bullmq": "5.78.1",
|
|
74
76
|
"compare-versions": "6.1.1",
|
|
75
77
|
"dotenv": "17.4.2",
|
|
76
78
|
"encoding-japanese": "2.2.0",
|
|
@@ -84,31 +86,31 @@
|
|
|
84
86
|
"html-to-text": "10.0.0",
|
|
85
87
|
"ical.js": "1.5.0",
|
|
86
88
|
"iconv-lite": "0.7.2",
|
|
87
|
-
"imapflow": "1.4.
|
|
89
|
+
"imapflow": "1.4.1",
|
|
88
90
|
"ioredfour": "1.4.1",
|
|
89
91
|
"ioredis": "5.11.1",
|
|
90
92
|
"ipaddr.js": "2.4.0",
|
|
91
|
-
"joi": "17.13.
|
|
93
|
+
"joi": "17.13.4",
|
|
92
94
|
"jquery": "4.0.0",
|
|
93
95
|
"libbase64": "1.3.0",
|
|
94
96
|
"libmime": "5.3.8",
|
|
95
97
|
"libqp": "2.1.1",
|
|
96
98
|
"license-checker": "25.0.1",
|
|
97
|
-
"mailparser": "3.9.
|
|
99
|
+
"mailparser": "3.9.10",
|
|
98
100
|
"marked": "9.1.6",
|
|
99
101
|
"minimist": "1.2.8",
|
|
100
102
|
"msgpack5": "6.0.2",
|
|
101
103
|
"murmurhash": "2.0.1",
|
|
102
104
|
"nanoid": "3.3.8",
|
|
103
|
-
"nodemailer": "
|
|
105
|
+
"nodemailer": "9.0.0",
|
|
104
106
|
"pino": "10.3.1",
|
|
105
107
|
"popper.js": "1.16.1",
|
|
106
108
|
"prom-client": "15.1.3",
|
|
107
109
|
"psl": "1.15.0",
|
|
108
|
-
"pubface": "1.1.
|
|
110
|
+
"pubface": "1.1.3",
|
|
109
111
|
"punycode.js": "2.3.1",
|
|
110
112
|
"qrcode": "1.5.4",
|
|
111
|
-
"smtp-server": "3.
|
|
113
|
+
"smtp-server": "3.19.0",
|
|
112
114
|
"socks": "2.8.9",
|
|
113
115
|
"speakeasy": "2.0.0",
|
|
114
116
|
"startbootstrap-sb-admin-2": "3.3.7",
|
|
@@ -118,14 +120,14 @@
|
|
|
118
120
|
},
|
|
119
121
|
"devDependencies": {
|
|
120
122
|
"@eslint/js": "10.0.1",
|
|
123
|
+
"acorn": "8.17.0",
|
|
124
|
+
"acorn-walk": "8.3.5",
|
|
121
125
|
"chai": "4.3.10",
|
|
122
126
|
"eerawlog": "1.5.3",
|
|
123
|
-
"eslint": "10.
|
|
127
|
+
"eslint": "10.5.0",
|
|
124
128
|
"grunt": "1.6.2",
|
|
125
129
|
"grunt-cli": "1.5.0",
|
|
126
130
|
"grunt-shell-spawn": "0.5.0",
|
|
127
|
-
"grunt-wait": "0.3.0",
|
|
128
|
-
"jsxgettext": "0.11.0",
|
|
129
131
|
"pino-pretty": "13.0.0",
|
|
130
132
|
"prettier": "3.8.4",
|
|
131
133
|
"resedit": "3.0.2",
|
|
@@ -157,6 +159,8 @@
|
|
|
157
159
|
"node_modules/@postalsys/joi-messages/translations/*",
|
|
158
160
|
"node_modules/@postalsys/bounce-classifier/model/**/*",
|
|
159
161
|
"node_modules/jsdom/lib/jsdom/browser/default-stylesheet.css",
|
|
162
|
+
"node_modules/nodemailer/lib/well-known/*.json",
|
|
163
|
+
"node_modules/nodemailer/package.json",
|
|
160
164
|
"LICENSE_EMAILENGINE.txt",
|
|
161
165
|
"version-info.json",
|
|
162
166
|
"sbom.json"
|