emailengine-app 2.69.0 → 2.71.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/.github/workflows/deploy.yml +6 -3
  2. package/.github/workflows/release.yaml +2 -0
  3. package/.github/workflows/test.yml +73 -12
  4. package/.ncurc.js +3 -3
  5. package/CHANGELOG.md +37 -0
  6. package/Gruntfile.js +21 -23
  7. package/bin/emailengine.js +8 -1
  8. package/config/default.toml +5 -0
  9. package/config/test.toml +5 -0
  10. package/data/google-crawlers.json +1 -1
  11. package/getswagger.sh +44 -4
  12. package/gettext-extract.js +163 -0
  13. package/lib/account.js +104 -72
  14. package/lib/api-routes/account-routes.js +231 -71
  15. package/lib/api-routes/blocklist-routes.js +25 -18
  16. package/lib/api-routes/chat-routes.js +32 -14
  17. package/lib/api-routes/delivery-test-routes.js +30 -5
  18. package/lib/api-routes/export-routes.js +27 -2
  19. package/lib/api-routes/gateway-routes.js +63 -12
  20. package/lib/api-routes/license-routes.js +18 -4
  21. package/lib/api-routes/mailbox-routes.js +33 -7
  22. package/lib/api-routes/message-routes.js +291 -145
  23. package/lib/api-routes/oauth2-app-routes.js +90 -24
  24. package/lib/api-routes/outbox-routes.js +16 -4
  25. package/lib/api-routes/pubsub-routes.js +8 -4
  26. package/lib/api-routes/route-helpers.js +14 -1
  27. package/lib/api-routes/settings-routes.js +51 -25
  28. package/lib/api-routes/stats-routes.js +37 -3
  29. package/lib/api-routes/submit-routes.js +31 -42
  30. package/lib/api-routes/template-routes.js +54 -21
  31. package/lib/api-routes/token-routes.js +67 -67
  32. package/lib/api-routes/webhook-route-routes.js +37 -8
  33. package/lib/autodetect-imap-settings.js +0 -2
  34. package/lib/consts.js +5 -0
  35. package/lib/document-store.js +22 -1
  36. package/lib/email-client/base-client.js +31 -8
  37. package/lib/email-client/gmail-client.js +119 -112
  38. package/lib/email-client/imap/mailbox.js +2 -2
  39. package/lib/email-client/imap/subconnection.js +0 -1
  40. package/lib/email-client/imap/sync-operations.js +1 -1
  41. package/lib/email-client/imap-client.js +36 -17
  42. package/lib/email-client/notification-handler.js +3 -6
  43. package/lib/email-client/outlook-client.js +49 -62
  44. package/lib/export.js +49 -1
  45. package/lib/feature-flags.js +8 -2
  46. package/lib/gateway.js +4 -9
  47. package/lib/get-raw-email.js +5 -5
  48. package/lib/imapproxy/imap-core/lib/imap-connection.js +0 -1
  49. package/lib/license-beacon.js +367 -0
  50. package/lib/logger.js +35 -22
  51. package/lib/metrics-collector.js +0 -2
  52. package/lib/oauth2-apps.js +13 -4
  53. package/lib/outbox.js +24 -40
  54. package/lib/redis-operations.js +1 -1
  55. package/lib/routes-ui.js +2 -1
  56. package/lib/schemas.js +403 -83
  57. package/lib/sentry.js +139 -0
  58. package/lib/settings.js +9 -3
  59. package/lib/stream-encrypt.js +1 -1
  60. package/lib/templates.js +1 -1
  61. package/lib/tokens.js +5 -3
  62. package/lib/tools.js +28 -6
  63. package/lib/ui-routes/account-routes.js +7 -4
  64. package/lib/ui-routes/admin-config-routes.js +20 -6
  65. package/lib/ui-routes/document-store-routes.js +7 -1
  66. package/lib/ui-routes/oauth-config-routes.js +0 -2
  67. package/lib/ui-routes/route-helpers.js +0 -2
  68. package/lib/ui-routes/unsubscribe-routes.js +0 -2
  69. package/lib/webhooks.js +8 -4
  70. package/package.json +23 -19
  71. package/sbom.json +1 -1
  72. package/server.js +38 -31
  73. package/static/licenses.html +171 -391
  74. package/translations/de.mo +0 -0
  75. package/translations/de.po +154 -142
  76. package/translations/et.mo +0 -0
  77. package/translations/et.po +129 -131
  78. package/translations/fr.mo +0 -0
  79. package/translations/fr.po +133 -136
  80. package/translations/ja.mo +0 -0
  81. package/translations/ja.po +126 -129
  82. package/translations/messages.pot +107 -107
  83. package/translations/nl.mo +0 -0
  84. package/translations/nl.po +128 -130
  85. package/translations/pl.mo +0 -0
  86. package/translations/pl.po +125 -128
  87. package/update-info.sh +19 -1
  88. package/views/config/logging.hbs +48 -0
  89. package/views/dashboard.hbs +22 -0
  90. package/workers/api.js +33 -37
  91. package/workers/documents.js +2 -22
  92. package/workers/export.js +73 -92
  93. package/workers/imap-proxy.js +3 -23
  94. package/workers/imap.js +2 -22
  95. package/workers/smtp.js +2 -22
  96. package/workers/submit.js +6 -24
  97. package/workers/webhooks.js +2 -22
package/lib/sentry.js ADDED
@@ -0,0 +1,139 @@
1
+ 'use strict';
2
+
3
+ const packageData = require('../package.json');
4
+ const logger = require('./logger');
5
+ const { readEnvValue } = require('./tools');
6
+ const { COMMUNITY_SENTRY_DSN } = require('./consts');
7
+
8
+ const SENTRY_SETTINGS_CHECK_INTERVAL = 60 * 1000;
9
+
10
+ let Sentry;
11
+ let workerName;
12
+ let activeDsn = false;
13
+
14
+ function startSentry(dsn) {
15
+ // require lazily, the SDK loads several hundred modules in every worker thread,
16
+ // so only pay that cost when error tracking is actually enabled
17
+ if (!Sentry) {
18
+ Sentry = require('@sentry/node');
19
+ }
20
+
21
+ Sentry.init({
22
+ dsn,
23
+ release: packageData.version,
24
+ // Error capture only: skip the OpenTelemetry setup and the default
25
+ // integrations that patch http/fetch/console on hot paths. Sentry's own
26
+ // uncaughtException/unhandledRejection integrations are left out on
27
+ // purpose - they do not run in worker threads and do not guarantee a
28
+ // process exit, so lib/logger.js owns reporting and exiting instead.
29
+ skipOpenTelemetrySetup: true,
30
+ defaultIntegrations: false,
31
+ integrations: [
32
+ Sentry.eventFiltersIntegration(),
33
+ Sentry.functionToStringIntegration(),
34
+ Sentry.linkedErrorsIntegration(),
35
+ Sentry.contextLinesIntegration(),
36
+ Sentry.nodeContextIntegration(),
37
+ Sentry.modulesIntegration()
38
+ ],
39
+ initialScope: {
40
+ tags: { worker: workerName }
41
+ }
42
+ });
43
+
44
+ logger.notifyError = (err, opts) => {
45
+ let captureContext = {};
46
+ if (opts?.user) {
47
+ captureContext.user = { id: `${opts.user}` };
48
+ }
49
+ if (opts?.meta && Object.keys(opts.meta).length) {
50
+ captureContext.contexts = { ee: opts.meta };
51
+ }
52
+ Sentry.captureException(err, captureContext);
53
+ };
54
+
55
+ // the global exception handlers in lib/logger.js wait for this before exiting
56
+ logger.flushNotifications = () => Sentry.flush(2000);
57
+ }
58
+
59
+ // Tag all events with the installation identity, so reports from different
60
+ // EmailEngine instances sharing the same DSN can be told apart. Read once per
61
+ // SDK start - a license swapped at runtime is reflected after the next restart.
62
+ async function applyIdentityTags() {
63
+ const settings = require('./settings');
64
+ let { serviceId, tract } = await settings.getMulti('serviceId', 'tract');
65
+ let tags = {};
66
+ if (serviceId) {
67
+ tags.instance = serviceId;
68
+ }
69
+ if (tract?.key) {
70
+ tags.license = tract.key;
71
+ }
72
+ Sentry.getGlobalScope().setTags(tags);
73
+ }
74
+
75
+ async function applySentryState(dsn) {
76
+ dsn = dsn || false;
77
+ if (dsn === activeDsn) {
78
+ return;
79
+ }
80
+
81
+ if (activeDsn) {
82
+ delete logger.notifyError;
83
+ delete logger.flushNotifications;
84
+ await Sentry.close(2000);
85
+ logger.info({ msg: 'Disabled Sentry error reporting', worker: workerName });
86
+ }
87
+
88
+ if (dsn) {
89
+ startSentry(dsn);
90
+ logger.info({ msg: 'Enabled Sentry error reporting', worker: workerName });
91
+
92
+ // identity lookup failure must not break error reporting, events just lack the tags
93
+ applyIdentityTags().catch(err => {
94
+ logger.error({ msg: 'Failed to apply Sentry identity tags', worker: workerName, err });
95
+ });
96
+ }
97
+
98
+ activeDsn = dsn;
99
+ }
100
+
101
+ async function checkSentrySettings() {
102
+ const settings = require('./settings');
103
+ let { sentryEnabled, sentryDsn } = await settings.getMulti('sentryEnabled', 'sentryDsn');
104
+ await applySentryState(sentryEnabled ? sentryDsn || COMMUNITY_SENTRY_DSN : false);
105
+ }
106
+
107
+ // Initialize Sentry error tracking. If the SENTRY_DSN environment variable is set,
108
+ // it pins the configuration and runtime settings are ignored. Otherwise the
109
+ // `sentryEnabled` and `sentryDsn` settings are applied at runtime and re-checked
110
+ // periodically, so error reporting can be toggled from the admin UI without a
111
+ // restart. While disabled, logger.notifyError stays undefined and the global
112
+ // exception handlers in lib/logger.js exit without waiting for a delivery flush.
113
+ function initSentry(worker) {
114
+ workerName = worker;
115
+
116
+ let envDsn = readEnvValue('SENTRY_DSN');
117
+ if (envDsn) {
118
+ applySentryState(envDsn).catch(err => {
119
+ logger.error({ msg: 'Failed to initialize Sentry', worker: workerName, err });
120
+ });
121
+ return;
122
+ }
123
+
124
+ // Settings changes are detected by polling instead of the {cmd: 'settings'}
125
+ // broadcast because that broadcast does not reach all the threads that
126
+ // initialize Sentry (smtp, imap-proxy, documents, and the main thread).
127
+ let checkSettings = async () => {
128
+ try {
129
+ await checkSentrySettings();
130
+ } catch (err) {
131
+ logger.error({ msg: 'Failed to apply Sentry settings', worker: workerName, err });
132
+ }
133
+ setTimeout(checkSettings, SENTRY_SETTINGS_CHECK_INTERVAL).unref();
134
+ };
135
+
136
+ setImmediate(checkSettings);
137
+ }
138
+
139
+ module.exports = { initSentry };
package/lib/settings.js CHANGED
@@ -13,9 +13,12 @@ config.service = config.service || {};
13
13
  const ENCRYPTED_KEYS = [
14
14
  'gmailClientSecret',
15
15
  'outlookClientSecret',
16
+ 'mailRuClientSecret',
16
17
  'cookiePassword',
17
18
  'smtpServerPassword',
19
+ 'imapProxyServerPassword',
18
20
  'serviceSecret',
21
+ 'serviceKey',
19
22
  'gmailServiceKey',
20
23
  'gmailServiceExternalAccount',
21
24
  'documentStorePassword',
@@ -100,6 +103,9 @@ module.exports = {
100
103
  formatSettingValue(key, value) {
101
104
  switch (key) {
102
105
  case 'serviceUrl': {
106
+ if (!value) {
107
+ return value;
108
+ }
103
109
  let urlObj = new URL(value);
104
110
  return urlObj.origin;
105
111
  }
@@ -155,9 +161,9 @@ module.exports = {
155
161
  }
156
162
  }
157
163
 
158
- if (key in ['generateEmailSummary', 'openAiGenerateEmbeddings'] && value) {
159
- // make sure `notifyText` is enabled
160
- return await redis.hset(`${REDIS_PREFIX}settings`, 'notifyText', 'true');
164
+ if (['generateEmailSummary', 'openAiGenerateEmbeddings'].includes(key) && formattedValue) {
165
+ // AI processing needs access to message text content, so make sure `notifyText` is enabled as well
166
+ await module.exports.set('notifyText', true);
161
167
  }
162
168
 
163
169
  if (/^openAi/.test(key) || key === 'generateEmailSummary') {
@@ -162,7 +162,7 @@ class DecryptStream extends Transform {
162
162
  decrypted = Buffer.concat([decipher.update(encryptedData), decipher.final()]);
163
163
  } catch (err) {
164
164
  if (err.message.includes('auth')) {
165
- throw new Error('Decryption failed: invalid secret or corrupted data');
165
+ throw new Error('Decryption failed: invalid secret or corrupted data', { cause: err });
166
166
  }
167
167
  throw err;
168
168
  }
package/lib/templates.js CHANGED
@@ -53,7 +53,7 @@ class TemplateHandler {
53
53
  let templateMeta = msgpack.decode(entry);
54
54
  response.templates.push(templateMeta);
55
55
  } catch (err) {
56
- logger.error({ msg: 'Failed to process template', entry: entry.toString('base64') });
56
+ logger.error({ msg: 'Failed to process template', entry: entry && entry.toString('base64'), err });
57
57
  continue;
58
58
  }
59
59
  }
package/lib/tokens.js CHANGED
@@ -325,16 +325,18 @@ module.exports = {
325
325
  let lastEntry = false;
326
326
  for (let i = 0; i < detailList.length; i++) {
327
327
  let entry = detailList[i];
328
+ // Each token occupies two consecutive multi results (data + access info)
329
+ let tokenHash = list[Math.floor(i / 2)];
328
330
  if (i % 2 === 0) {
329
331
  lastEntry = false;
330
332
  if (entry[1]) {
331
333
  try {
332
334
  let tokenData = msgpack.decode(entry[1]);
333
335
  tokenData.created = new Date(tokenData.created);
334
- lastEntry = Object.assign({ id: list[response.tokens.length] }, tokenData);
336
+ lastEntry = Object.assign({ id: tokenHash }, tokenData);
335
337
  response.tokens.push(lastEntry);
336
338
  } catch (err) {
337
- logger.error({ msg: 'Failed to process token data', hash: list[response.tokens.length], err });
339
+ logger.error({ msg: 'Failed to process token data', hash: tokenHash, err });
338
340
  }
339
341
  }
340
342
  } else if (lastEntry && entry[1]) {
@@ -343,7 +345,7 @@ module.exports = {
343
345
  accessData.time = accessData.time ? new Date(accessData.time) : null;
344
346
  lastEntry.access = accessData;
345
347
  } catch (err) {
346
- logger.error({ msg: 'Failed to process token data', hash: list[i], err });
348
+ logger.error({ msg: 'Failed to process token data', hash: tokenHash, err });
347
349
  }
348
350
  }
349
351
  }
package/lib/tools.js CHANGED
@@ -1,7 +1,5 @@
1
1
  /* eslint no-bitwise: 0 */
2
2
 
3
- // NB! This file is processed by gettext parser and can not use newer syntax like ?.
4
-
5
3
  'use strict';
6
4
 
7
5
  const msgpack = require('msgpack5')();
@@ -59,7 +57,26 @@ const AGENT_OPTS = {
59
57
  const RETRY_OPTS = {
60
58
  maxRetries: URL_FETCH_RETRY_MAX,
61
59
  methods: ['GET', 'PUT', 'HEAD', 'OPTIONS', 'DELETE', 'POST'],
62
- statusCodes: [429] // do not retry 5xx errors
60
+ statusCodes: [429], // do not retry 5xx errors
61
+ // undici does not retry transient DNS failures (EAI_AGAIN) or socket connect
62
+ // timeouts (ETIMEDOUT) by default, so a name-resolution blip bubbles up as a hard
63
+ // `TypeError: fetch failed`. Add them to the retry list. Passing errorCodes REPLACES
64
+ // undici's defaults, so its default codes are re-listed here.
65
+ errorCodes: [
66
+ // undici defaults
67
+ 'ECONNRESET',
68
+ 'ECONNREFUSED',
69
+ 'ENOTFOUND',
70
+ 'ENETDOWN',
71
+ 'ENETUNREACH',
72
+ 'EHOSTDOWN',
73
+ 'EHOSTUNREACH',
74
+ 'EPIPE',
75
+ 'UND_ERR_SOCKET',
76
+ // additions: transient DNS / socket connect-timeout failures
77
+ 'EAI_AGAIN',
78
+ 'ETIMEDOUT'
79
+ ]
63
80
  };
64
81
 
65
82
  // Shared mutable object -- consumers import the object reference and access
@@ -1517,9 +1534,9 @@ MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEV3QUiYsp13nD9suD1/ZkEXnuMoSg
1517
1534
  .hget(`${REDIS_PREFIX}bull:${queue}:meta`, 'paused')
1518
1535
  .exec();
1519
1536
  if (resActive[0] || resDelayed[0] || resWaiting[0] || resPaused[0] || resMeta[0]) {
1520
- // counting failed
1537
+ // counting failed, skip this queue but keep the rest of the stats response
1521
1538
  logger.error({ msg: 'Failed to count queue length', queue, active: resActive, delayed: resDelayed, waiting: resWaiting });
1522
- return false;
1539
+ continue;
1523
1540
  }
1524
1541
  queues[queue] = {
1525
1542
  active: Number(resActive[1]) || 0,
@@ -2200,6 +2217,7 @@ vWuuhT9ely8AUX2F
2200
2217
  mF3GOI+Ev7mJODtG
2201
2218
  nQNwPlZ+tyx24XVo
2202
2219
  olObiSmdzVaMp7lH
2220
+ SruXHv6plKL9YuAW
2203
2221
  cyp1GKV4Cl+eC8G/
2204
2222
  Q1y/TzbtUQCqnotR
2205
2223
  59Q2qOkuyTLzQDhd
@@ -2223,9 +2241,13 @@ EsbWnuADOq9qe/EZ
2223
2241
  YfUSxQtq1+kpwW/W
2224
2242
  QodzxvoJ6NPGhCgW
2225
2243
  5/VK+O950efBio0t
2226
- RA/lrkwxcTX80nxX
2227
2244
  b85BMAFRHn10uX8y
2228
2245
  vV2RgfFH2YFTYsly
2246
+ zcmIO3obSzUVHGtF
2247
+ tzsA68uFVAffHmec
2248
+ DkCU3OpaSa+rIVfU
2249
+ dcDjqpxVxivT46Rl
2250
+ QSgNcpNukC9CBNfe
2229
2251
  `
2230
2252
  .split(/\r?\n/)
2231
2253
  .map(l => l.trim())
@@ -1,7 +1,5 @@
1
1
  'use strict';
2
2
 
3
- // NB! This file is processed by the gettext parser (npm run gettext) and can not use newer syntax like ?.
4
-
5
3
  // Admin UI routes for account management: /admin/accounts (listing), /admin/accounts/new and
6
4
  // /accounts/new* (the add-account wizard incl. IMAP autoconfig/test/server steps), and
7
5
  // /admin/accounts/{account}* (view, edit, delete, reconnect, sync, logs, browse). Extracted
@@ -33,7 +31,7 @@ const {
33
31
  readEnvValue,
34
32
  failAction
35
33
  } = require('../tools');
36
- const { settingsSchema, accountIdSchema, defaultAccountTypeSchema } = require('../schemas');
34
+ const { settingsSchema, accountIdSchema, defaultAccountTypeSchema, ACCOUNT_DISPLAY_STATES } = require('../schemas');
37
35
  const { formatAccountData, cachedTemplates } = require('./route-helpers');
38
36
 
39
37
  const { REDIS_PREFIX, DEFAULT_PAGE_SIZE, NONCE_BYTES, MAX_FORM_TTL, DEFAULT_MAX_LOG_LINES } = consts;
@@ -179,6 +177,11 @@ function init(args) {
179
177
  label: 'Disconnected'
180
178
  },
181
179
 
180
+ {
181
+ state: 'paused',
182
+ label: 'Paused'
183
+ },
184
+
182
185
  {
183
186
  state: 'authenticationError',
184
187
  label: 'Authentication failed'
@@ -252,7 +255,7 @@ function init(args) {
252
255
  state: Joi.string()
253
256
  .trim()
254
257
  .empty('')
255
- .valid('init', 'syncing', 'connecting', 'connected', 'authenticationError', 'connectError', 'unset', 'disconnected')
258
+ .valid(...ACCOUNT_DISPLAY_STATES)
256
259
  .example('connected')
257
260
  .description('Filter accounts by state')
258
261
  .label('AccountState')
@@ -1,7 +1,5 @@
1
1
  'use strict';
2
2
 
3
- // NB! This file is processed by the gettext parser (npm run gettext) and can not use newer syntax like ?.
4
-
5
3
  // Admin UI routes for the remaining /admin/config/* pages: webhooks, service URL/branding,
6
4
  // AI (OpenAI) settings, logging, and license. Extracted verbatim from lib/routes-ui.js.
7
5
  // The notificationTypes / configWebhooksSchema / configLoggingSchema consts, the
@@ -25,6 +23,7 @@ const timezonesList = require('timezones-list').default;
25
23
  const { redis, submitQueue, notifyQueue, documentsQueue } = require('../db');
26
24
  const { getByteSize, formatByteSize, getDuration, failAction, hasEnvValue, readEnvValue, httpAgent } = require('../tools');
27
25
  const { llmPreProcess } = require('../llm-pre-process');
26
+ const { documentStoreFeatureEnabled } = require('../document-store');
28
27
  const { locales } = require('../translations');
29
28
  const { settingsSchema } = require('../schemas');
30
29
  const { getOpenAiModels, OPEN_AI_MODELS, getExampleDocumentsPayloads } = require('./route-helpers');
@@ -131,7 +130,9 @@ for (let type of notificationTypes) {
131
130
 
132
131
  const configLoggingSchema = {
133
132
  all: Joi.boolean().truthy('Y', 'true', '1', 'on').falsy('N', 'false', 0, '').default(false).description('Enable logs for all accounts'),
134
- maxLogLines: Joi.number().integer().empty('').min(0).max(10000000).default(DEFAULT_MAX_LOG_LINES)
133
+ maxLogLines: Joi.number().integer().empty('').min(0).max(10000000).default(DEFAULT_MAX_LOG_LINES),
134
+ sentryEnabled: settingsSchema.sentryEnabled.default(false),
135
+ sentryDsn: settingsSchema.sentryDsn.default('')
135
136
  };
136
137
 
137
138
  async function getOpenAiError(gt) {
@@ -217,7 +218,7 @@ function init(args) {
217
218
  values,
218
219
 
219
220
  webhookErrorFlag: await settings.get('webhookErrorFlag'),
220
- documentStoreEnabled: (await settings.get('documentStoreEnabled')) || false
221
+ documentStoreEnabled: (documentStoreFeatureEnabled && (await settings.get('documentStoreEnabled'))) || false
221
222
  },
222
223
  {
223
224
  layout: 'app'
@@ -307,7 +308,7 @@ function init(args) {
307
308
  ),
308
309
 
309
310
  webhookErrorFlag: await settings.get('webhookErrorFlag'),
310
- documentStoreEnabled: (await settings.get('documentStoreEnabled')) || false
311
+ documentStoreEnabled: (documentStoreFeatureEnabled && (await settings.get('documentStoreEnabled'))) || false
311
312
  },
312
313
  {
313
314
  layout: 'app'
@@ -356,7 +357,7 @@ function init(args) {
356
357
  errors,
357
358
 
358
359
  webhookErrorFlag: await settings.get('webhookErrorFlag'),
359
- documentStoreEnabled: (await settings.get('documentStoreEnabled')) || false
360
+ documentStoreEnabled: (documentStoreFeatureEnabled && (await settings.get('documentStoreEnabled'))) || false
360
361
  },
361
362
  {
362
363
  layout: 'app'
@@ -1120,6 +1121,10 @@ return true;`
1120
1121
 
1121
1122
  values.accounts = [].concat(values.accounts || []).join('\n');
1122
1123
 
1124
+ let sentryValues = await settings.getMulti('sentryEnabled', 'sentryDsn');
1125
+ values.sentryEnabled = !!sentryValues.sentryEnabled;
1126
+ values.sentryDsn = sentryValues.sentryDsn || '';
1127
+
1123
1128
  return h.view(
1124
1129
  'config/logging',
1125
1130
  {
@@ -1127,6 +1132,8 @@ return true;`
1127
1132
  menuConfig: true,
1128
1133
  menuConfigLogging: true,
1129
1134
 
1135
+ sentryEnvManaged: !!readEnvValue('SENTRY_DSN'),
1136
+
1130
1137
  values
1131
1138
  },
1132
1139
  {
@@ -1148,6 +1155,13 @@ return true;`
1148
1155
  }
1149
1156
  };
1150
1157
 
1158
+ if (!readEnvValue('SENTRY_DSN')) {
1159
+ // the form renders these fields as disabled when the DSN is pinned by the
1160
+ // environment, so do not overwrite the stored values in that case
1161
+ data.sentryEnabled = !!request.payload.sentryEnabled;
1162
+ data.sentryDsn = request.payload.sentryDsn || '';
1163
+ }
1164
+
1151
1165
  for (let key of Object.keys(data)) {
1152
1166
  await settings.set(key, data[key]);
1153
1167
  }
@@ -16,7 +16,7 @@ const { REDIS_PREFIX } = require('../consts');
16
16
  const { failAction } = require('../tools');
17
17
  const { settingsSchema } = require('../schemas');
18
18
  const { defaultMappings } = require('../es');
19
- const { getESClient } = require('../document-store');
19
+ const { getESClient, documentStoreFeatureEnabled } = require('../document-store');
20
20
  const { getOpenAiModels, OPEN_AI_MODELS, getExampleDocumentsPayloads } = require('./route-helpers');
21
21
 
22
22
  const FIELD_TYPES = [
@@ -95,6 +95,12 @@ const configDocumentStoreSchema = {
95
95
  function init(args) {
96
96
  const { server } = args;
97
97
 
98
+ // Deprecated Document Store feature: when the gate is off, register no routes so that
99
+ // every /admin/config/document-store* page behaves like a regular 404.
100
+ if (!documentStoreFeatureEnabled) {
101
+ return;
102
+ }
103
+
98
104
  server.route({
99
105
  method: 'GET',
100
106
  path: '/admin/config/document-store',
@@ -1,7 +1,5 @@
1
1
  'use strict';
2
2
 
3
- // NB! This file is processed by the gettext parser (npm run gettext) and can not use newer syntax like ?.
4
-
5
3
  // Admin UI routes for OAuth2 application config (/admin/config/oauth*): listing apps,
6
4
  // per-app view/edit/delete, creating apps, adding accounts, provider subscriptions, and
7
5
  // app verification. Extracted verbatim from lib/routes-ui.js. AZURE_CLOUDS and the
@@ -1,7 +1,5 @@
1
1
  'use strict';
2
2
 
3
- // NB! This file is processed by the gettext parser (npm run gettext) and can not use newer syntax like ?.
4
-
5
3
  // Shared helpers used by more than one extracted UI route module - and still by
6
4
  // lib/routes-ui.js for the route groups not yet extracted. Lifting these here lets each
7
5
  // consumer import the single canonical copy instead of the monolith, so a route group can
@@ -1,7 +1,5 @@
1
1
  'use strict';
2
2
 
3
- // NB! This file is processed by the gettext parser (npm run gettext) and can not use newer syntax like ?.
4
-
5
3
  // Public (unauthenticated) subscription-management routes. These render the unsubscribe
6
4
  // landing page reached from the List-Unsubscribe link in outgoing messages and process
7
5
  // the subscribe/unsubscribe form submission. Extracted verbatim from lib/routes-ui.js.
package/lib/webhooks.js CHANGED
@@ -70,12 +70,12 @@ class WebhooksHandler {
70
70
 
71
71
  try {
72
72
  let webhookMeta = msgpack.decode(entry);
73
- if (webhookErrorFlag && typeof webhookErrorFlag === 'object' && !Object.keys(webhookErrorFlag)) {
73
+ if (webhookErrorFlag && typeof webhookErrorFlag === 'object' && !Object.keys(webhookErrorFlag).length) {
74
74
  webhookErrorFlag = null;
75
75
  }
76
76
  response.webhooks.push(Object.assign(webhookMeta, { tcount, webhookErrorFlag }));
77
77
  } catch (err) {
78
- logger.error({ msg: 'Failed to process webhook', entry: entry.toString('base64') });
78
+ logger.error({ msg: 'Failed to process webhook', entry: entry && entry.toString('base64'), err });
79
79
  continue;
80
80
  }
81
81
  }
@@ -398,7 +398,6 @@ class WebhooksHandler {
398
398
  v = Number(v) || 0;
399
399
  if (v !== this.handlerCacheV) {
400
400
  // changes detected
401
- v = this.handlerCacheV;
402
401
 
403
402
  let webhookIds = await this.redis.smembers(this.getWebhooksIndexKey());
404
403
  webhookIds = [].concat(webhookIds || []).sort((a, b) => -a.localeCompare(b));
@@ -418,7 +417,8 @@ class WebhooksHandler {
418
417
  this.handlerCache.push(handler);
419
418
  } else {
420
419
  // compare existing
421
- let webhookV = await this.redis.hget(this.getWebhooksContentKey(), `${webhookId}:v`);
420
+ // the per-route counter is stored as a string, existing.v is a number (see get())
421
+ let webhookV = Number(await this.redis.hget(this.getWebhooksContentKey(), `${webhookId}:v`)) || 0;
422
422
  if (existing.v !== webhookV) {
423
423
  // update
424
424
  for (let i = this.handlerCache.length - 1; i >= 0; i--) {
@@ -430,6 +430,10 @@ class WebhooksHandler {
430
430
  }
431
431
  }
432
432
  }
433
+
434
+ // mark the cache as current only after a successful refresh, so a failure above
435
+ // leaves the cache stale and the next call retries
436
+ this.handlerCacheV = v;
433
437
  }
434
438
 
435
439
  return this.handlerCache;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "emailengine-app",
3
- "version": "2.69.0",
3
+ "version": "2.71.0",
4
4
  "private": false,
5
5
  "productTitle": "EmailEngine",
6
6
  "description": "Email Sync Engine",
@@ -11,13 +11,15 @@
11
11
  "single": "EE_OPENAPI_VERBOSE=true EENGINE_LOG_RAW=true EENGINE_SECRET=your-encryption-key EENGINE_WORKERS=1 node --inspect server --dbs.redis='redis://127.0.0.1:6379/6' --api.port=7003 --api.host=0.0.0.0 | tee $HOME/ee.log.single.txt | pino-pretty",
12
12
  "gmail": "EE_OPENAPI_VERBOSE=true EENGINE_LOG_RAW=true EENGINE_SECRET=your-encryption-key EENGINE_WORKERS=2 EENGINE_CORS_ORIGIN='*' node --inspect server --dbs.redis='redis://127.0.0.1:6379/11' --api.port=7003 --api.host=0.0.0.0 | tee $HOME/ee.log.gmail.txt | pino-pretty",
13
13
  "test": "NODE_ENV=test grunt",
14
- "lint": "npx eslint lib/**/*.js workers/**/*.js test/**/*.js server.js Gruntfile.js",
14
+ "test:unit": "NODE_ENV=test grunt test-unit",
15
+ "test:integration": "NODE_ENV=test grunt test-integration",
16
+ "lint": "npx eslint 'lib/**/*.js' 'workers/**/*.js' 'test/**/*.js' server.js Gruntfile.js",
15
17
  "swagger": "./getswagger.sh",
16
18
  "build-source": "rm -rf node_modules && npm install && rm -rf node_modules && npm ci --omit=dev && rm -rf node_modules/ace-builds node_modules/@postalsys/ee-client && ./update-info.sh",
17
19
  "build-dist": "pkg --compress Brotli package.json && npm install && node winconf.js",
18
20
  "build-dist-fast": "pkg --debug package.json && npm install && node winconf.js",
19
21
  "licenses": "license-checker --excludePackages 'emailengine-app' --json | node list-generate.js > static/licenses.html",
20
- "gettext": "find ./views -name \"*.hbs\" -print0 | xargs -0 xgettext-template -L Handlebars -o translations/messages.pot --force-po && jsxgettext --parser-options '{\"ecmaVersion\": 2018}' workers/api.js lib/tools.js lib/autodetect-imap-settings.js lib/ui-routes/admin-entities-routes.js lib/ui-routes/account-routes.js lib/ui-routes/oauth-config-routes.js lib/ui-routes/admin-config-routes.js lib/ui-routes/route-helpers.js lib/ui-routes/unsubscribe-routes.js -j -o translations/messages.pot",
22
+ "gettext": "find ./views -name \"*.hbs\" -print0 | xargs -0 xgettext-template -L Handlebars -o translations/messages.pot --force-po && node gettext-extract.js",
21
23
  "prepare-docker": "echo \"EE_DOCKER_LEGACY=$EE_DOCKER_LEGACY\" >> system.env && cat system.env",
22
24
  "update": "rm -rf node_modules package-lock.json && ncu -u && npm install && ./copy-static-files.sh && npm run licenses && npm run gettext",
23
25
  "test-gmail-api": "node lib/email-client/gmail-client.js --dbs.redis=redis://127.0.0.1/11",
@@ -43,9 +45,8 @@
43
45
  },
44
46
  "homepage": "https://emailengine.app/",
45
47
  "dependencies": {
46
- "@bugsnag/js": "8.9.0",
47
- "@bull-board/api": "7.2.1",
48
- "@bull-board/hapi": "7.2.1",
48
+ "@bull-board/api": "8.0.0",
49
+ "@bull-board/hapi": "8.0.0",
49
50
  "@elastic/elasticsearch": "8.15.3",
50
51
  "@hapi/accept": "6.0.3",
51
52
  "@hapi/bell": "13.1.0",
@@ -57,20 +58,21 @@
57
58
  "@hapi/vision": "7.0.3",
58
59
  "@phc/pbkdf2": "1.1.14",
59
60
  "@postalsys/bounce-classifier": "3.0.0",
60
- "@postalsys/certs": "1.0.14",
61
+ "@postalsys/certs": "1.0.15",
61
62
  "@postalsys/ee-client": "1.3.0",
62
- "@postalsys/email-ai-tools": "1.13.5",
63
- "@postalsys/email-text-tools": "2.4.6",
63
+ "@postalsys/email-ai-tools": "1.13.6",
64
+ "@postalsys/email-text-tools": "2.4.7",
64
65
  "@postalsys/gettext": "4.1.1",
65
66
  "@postalsys/joi-messages": "1.0.5",
66
67
  "@postalsys/templates": "2.0.1",
68
+ "@sentry/node": "10.58.0",
67
69
  "@simplewebauthn/browser": "13.3.0",
68
70
  "@simplewebauthn/server": "13.3.1",
69
71
  "@zone-eu/mailsplit": "5.4.12",
70
72
  "@zone-eu/wild-config": "1.7.5",
71
73
  "ace-builds": "1.44.0",
72
74
  "base32.js": "0.1.0",
73
- "bullmq": "5.78.0",
75
+ "bullmq": "5.78.1",
74
76
  "compare-versions": "6.1.1",
75
77
  "dotenv": "17.4.2",
76
78
  "encoding-japanese": "2.2.0",
@@ -84,31 +86,31 @@
84
86
  "html-to-text": "10.0.0",
85
87
  "ical.js": "1.5.0",
86
88
  "iconv-lite": "0.7.2",
87
- "imapflow": "1.4.0",
89
+ "imapflow": "1.4.1",
88
90
  "ioredfour": "1.4.1",
89
91
  "ioredis": "5.11.1",
90
92
  "ipaddr.js": "2.4.0",
91
- "joi": "17.13.3",
93
+ "joi": "17.13.4",
92
94
  "jquery": "4.0.0",
93
95
  "libbase64": "1.3.0",
94
96
  "libmime": "5.3.8",
95
97
  "libqp": "2.1.1",
96
98
  "license-checker": "25.0.1",
97
- "mailparser": "3.9.9",
99
+ "mailparser": "3.9.10",
98
100
  "marked": "9.1.6",
99
101
  "minimist": "1.2.8",
100
102
  "msgpack5": "6.0.2",
101
103
  "murmurhash": "2.0.1",
102
104
  "nanoid": "3.3.8",
103
- "nodemailer": "8.0.10",
105
+ "nodemailer": "9.0.0",
104
106
  "pino": "10.3.1",
105
107
  "popper.js": "1.16.1",
106
108
  "prom-client": "15.1.3",
107
109
  "psl": "1.15.0",
108
- "pubface": "1.1.2",
110
+ "pubface": "1.1.3",
109
111
  "punycode.js": "2.3.1",
110
112
  "qrcode": "1.5.4",
111
- "smtp-server": "3.18.5",
113
+ "smtp-server": "3.19.0",
112
114
  "socks": "2.8.9",
113
115
  "speakeasy": "2.0.0",
114
116
  "startbootstrap-sb-admin-2": "3.3.7",
@@ -118,14 +120,14 @@
118
120
  },
119
121
  "devDependencies": {
120
122
  "@eslint/js": "10.0.1",
123
+ "acorn": "8.17.0",
124
+ "acorn-walk": "8.3.5",
121
125
  "chai": "4.3.10",
122
126
  "eerawlog": "1.5.3",
123
- "eslint": "10.4.1",
127
+ "eslint": "10.5.0",
124
128
  "grunt": "1.6.2",
125
129
  "grunt-cli": "1.5.0",
126
130
  "grunt-shell-spawn": "0.5.0",
127
- "grunt-wait": "0.3.0",
128
- "jsxgettext": "0.11.0",
129
131
  "pino-pretty": "13.0.0",
130
132
  "prettier": "3.8.4",
131
133
  "resedit": "3.0.2",
@@ -157,6 +159,8 @@
157
159
  "node_modules/@postalsys/joi-messages/translations/*",
158
160
  "node_modules/@postalsys/bounce-classifier/model/**/*",
159
161
  "node_modules/jsdom/lib/jsdom/browser/default-stylesheet.css",
162
+ "node_modules/nodemailer/lib/well-known/*.json",
163
+ "node_modules/nodemailer/package.json",
160
164
  "LICENSE_EMAILENGINE.txt",
161
165
  "version-info.json",
162
166
  "sbom.json"