emailengine-app 2.68.1 → 2.70.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. package/.github/workflows/deploy.yml +8 -3
  2. package/.github/workflows/release.yaml +6 -0
  3. package/CHANGELOG.md +59 -0
  4. package/Gruntfile.js +3 -1
  5. package/config/default.toml +2 -0
  6. package/data/google-crawlers.json +7 -1
  7. package/getswagger.sh +40 -4
  8. package/gettext-extract.js +163 -0
  9. package/lib/account.js +135 -72
  10. package/lib/api-routes/account-routes.js +684 -106
  11. package/lib/api-routes/blocklist-routes.js +344 -0
  12. package/lib/api-routes/chat-routes.js +32 -14
  13. package/lib/api-routes/delivery-test-routes.js +346 -0
  14. package/lib/api-routes/export-routes.js +28 -14
  15. package/lib/api-routes/gateway-routes.js +427 -0
  16. package/lib/api-routes/license-routes.js +156 -0
  17. package/lib/api-routes/mailbox-routes.js +344 -0
  18. package/lib/api-routes/message-routes.js +221 -187
  19. package/lib/api-routes/oauth2-app-routes.js +697 -0
  20. package/lib/api-routes/outbox-routes.js +185 -0
  21. package/lib/api-routes/pubsub-routes.js +102 -0
  22. package/lib/api-routes/route-helpers.js +58 -0
  23. package/lib/api-routes/settings-routes.js +357 -0
  24. package/lib/api-routes/stats-routes.js +111 -0
  25. package/lib/api-routes/submit-routes.js +461 -0
  26. package/lib/api-routes/template-routes.js +60 -75
  27. package/lib/api-routes/token-routes.js +297 -0
  28. package/lib/api-routes/webhook-route-routes.js +181 -0
  29. package/lib/autodetect-imap-settings.js +0 -2
  30. package/lib/consts.js +5 -0
  31. package/lib/email-client/base-client.js +28 -6
  32. package/lib/email-client/gmail-client.js +133 -112
  33. package/lib/email-client/imap/mailbox.js +34 -11
  34. package/lib/email-client/imap/subconnection.js +20 -13
  35. package/lib/email-client/imap/sync-operations.js +131 -3
  36. package/lib/email-client/imap-client.js +152 -75
  37. package/lib/email-client/notification-handler.js +1 -4
  38. package/lib/email-client/outlook-client.js +134 -75
  39. package/lib/export.js +97 -20
  40. package/lib/feature-flags.js +2 -2
  41. package/lib/gateway.js +4 -9
  42. package/lib/get-raw-email.js +5 -5
  43. package/lib/imapproxy/imap-core/lib/commands/starttls.js +18 -0
  44. package/lib/imapproxy/imap-core/lib/imap-command.js +6 -1
  45. package/lib/imapproxy/imap-core/lib/imap-connection.js +106 -24
  46. package/lib/imapproxy/imap-core/lib/imap-server.js +24 -0
  47. package/lib/imapproxy/imap-core/lib/imap-stream.js +26 -0
  48. package/lib/logger.js +24 -21
  49. package/lib/message-port-stream.js +113 -16
  50. package/lib/metrics-collector.js +0 -2
  51. package/lib/oauth2-apps.js +13 -4
  52. package/lib/outbox.js +24 -40
  53. package/lib/redis-operations.js +1 -1
  54. package/lib/reject-worker-calls.js +42 -0
  55. package/lib/routes-ui.js +37 -8778
  56. package/lib/schemas.js +429 -84
  57. package/lib/sentry.js +139 -0
  58. package/lib/settings.js +9 -3
  59. package/lib/stream-encrypt.js +1 -1
  60. package/lib/templates.js +1 -1
  61. package/lib/tokens.js +5 -3
  62. package/lib/tools.js +70 -4
  63. package/lib/ui-routes/account-routes.js +45 -212
  64. package/lib/ui-routes/admin-config-routes.js +928 -489
  65. package/lib/ui-routes/admin-entities-routes.js +1 -0
  66. package/lib/ui-routes/auth-routes.js +1339 -0
  67. package/lib/ui-routes/dashboard-routes.js +188 -0
  68. package/lib/ui-routes/document-store-routes.js +800 -0
  69. package/lib/ui-routes/export-routes.js +217 -0
  70. package/lib/ui-routes/internals-routes.js +354 -0
  71. package/lib/ui-routes/network-config-routes.js +759 -0
  72. package/lib/ui-routes/{oauth-routes.js → oauth-config-routes.js} +369 -91
  73. package/lib/ui-routes/route-helpers.js +314 -0
  74. package/lib/ui-routes/smtp-test-routes.js +236 -0
  75. package/lib/ui-routes/unsubscribe-routes.js +232 -0
  76. package/lib/webhook-request.js +36 -0
  77. package/lib/webhooks.js +8 -4
  78. package/package.json +13 -12
  79. package/sbom.json +1 -1
  80. package/server.js +222 -39
  81. package/static/licenses.html +160 -300
  82. package/translations/messages.pot +112 -132
  83. package/update-info.sh +19 -1
  84. package/views/config/logging.hbs +48 -0
  85. package/views/dashboard.hbs +7 -26
  86. package/views/internals/index.hbs +15 -0
  87. package/views/tokens/index.hbs +9 -0
  88. package/workers/api.js +200 -4424
  89. package/workers/documents.js +2 -22
  90. package/workers/export.js +103 -104
  91. package/workers/imap-proxy.js +3 -23
  92. package/workers/imap.js +32 -36
  93. package/workers/smtp.js +2 -22
  94. package/workers/submit.js +26 -35
  95. package/workers/webhooks.js +9 -43
@@ -1,205 +1,43 @@
1
1
  'use strict';
2
2
 
3
- const Boom = require('@hapi/boom');
3
+ // Admin UI routes for account management: /admin/accounts (listing), /admin/accounts/new and
4
+ // /accounts/new* (the add-account wizard incl. IMAP autoconfig/test/server steps), and
5
+ // /admin/accounts/{account}* (view, edit, delete, reconnect, sync, logs, browse). Extracted
6
+ // verbatim from lib/routes-ui.js. DISABLE_MESSAGE_BROWSER and the getMailboxListing helper move
7
+ // with the routes (only these account pages use them).
8
+
4
9
  const Joi = require('joi');
5
10
  const crypto = require('crypto');
6
- const util = require('util');
7
- const psl = require('psl');
11
+ const Boom = require('@hapi/boom');
8
12
 
9
13
  const settings = require('../settings');
14
+ const consts = require('../consts');
10
15
  const tokens = require('../tokens');
11
16
  const { redis, documentsQueue } = require('../db');
17
+ const getSecret = require('../get-secret');
18
+ const capa = require('../capa');
19
+ const { Account } = require('../account');
20
+ const { Gateway } = require('../gateway');
21
+ const { autodetectImapSettings } = require('../autodetect-imap-settings');
22
+ const { oauth2Apps, oauth2ProviderData, SERVICE_ACCOUNT_PROVIDERS } = require('../oauth2-apps');
12
23
  const {
13
- failAction,
14
- verifyAccountInfo,
15
- getLogs,
16
- flattenObjectKeys,
17
- getSignedFormData,
18
24
  getServiceHostname,
25
+ getSignedFormData,
19
26
  parseSignedFormData,
27
+ getLogs,
28
+ verifyAccountInfo,
29
+ flattenObjectKeys,
20
30
  getBoolean,
21
- readEnvValue
31
+ readEnvValue,
32
+ failAction
22
33
  } = require('../tools');
23
- const { Account } = require('../account');
24
- const { Gateway } = require('../gateway');
25
- const { oauth2Apps, oauth2ProviderData, SERVICE_ACCOUNT_PROVIDERS } = require('../oauth2-apps');
26
- const { autodetectImapSettings } = require('../autodetect-imap-settings');
27
- const getSecret = require('../get-secret');
28
- const capa = require('../capa');
29
- const consts = require('../consts');
30
- const { settingsSchema, accountIdSchema, defaultAccountTypeSchema } = require('../schemas');
31
- const fs = require('fs');
32
- const pathlib = require('path');
34
+ const { settingsSchema, accountIdSchema, defaultAccountTypeSchema, ACCOUNT_DISPLAY_STATES } = require('../schemas');
35
+ const { formatAccountData, cachedTemplates } = require('./route-helpers');
33
36
 
34
- const { DEFAULT_MAX_LOG_LINES, DEFAULT_PAGE_SIZE, REDIS_PREFIX, MAX_FORM_TTL, NONCE_BYTES } = consts;
37
+ const { REDIS_PREFIX, DEFAULT_PAGE_SIZE, NONCE_BYTES, MAX_FORM_TTL, DEFAULT_MAX_LOG_LINES } = consts;
35
38
 
36
39
  const DISABLE_MESSAGE_BROWSER = getBoolean(readEnvValue('EENGINE_DISABLE_MESSAGE_BROWSER'));
37
40
 
38
- const cachedTemplates = {
39
- testSend: fs.readFileSync(pathlib.join(__dirname, '..', '..', 'views', 'partials', 'test_send.hbs'), 'utf-8')
40
- };
41
-
42
- function formatAccountData(account, gt) {
43
- account.type = {};
44
-
45
- if (account.oauth2 && account.oauth2.app) {
46
- let providerData = oauth2ProviderData(account.oauth2.app.provider);
47
- account.type = providerData;
48
- } else if (account.oauth2 && account.oauth2.provider) {
49
- account.type = oauth2ProviderData(account.oauth2.provider);
50
- } else if (account.imap && !account.imap.disabled) {
51
- account.type.icon = 'fa fa-envelope-square';
52
- account.type.name = 'IMAP';
53
- account.type.comment = psl.get(account.imap.host) || account.imap.host;
54
- } else if (account.smtp) {
55
- account.type.icon = 'fa fa-paper-plane';
56
- account.type.name = 'SMTP';
57
- account.type.comment = psl.get(account.smtp.host) || account.smtp.host;
58
- } else if (account.oauth2 && account.oauth2.auth && account.oauth2.auth.delegatedAccount) {
59
- account.type.icon = 'fa fa-arrow-alt-circle-right';
60
- account.type.name = gt.gettext('Delegated');
61
- account.type.comment = util.format(gt.gettext('Using credentials from "%s"'), account.oauth2.auth.delegatedAccount);
62
- } else {
63
- account.type.name = 'N/A';
64
- }
65
-
66
- switch (account.state) {
67
- case 'init':
68
- account.stateLabel = {
69
- type: 'info',
70
- name: 'Initializing',
71
- spinner: true
72
- };
73
- break;
74
-
75
- case 'connecting':
76
- account.stateLabel = {
77
- type: 'info',
78
- name: 'Connecting'
79
- };
80
- break;
81
-
82
- case 'syncing':
83
- account.stateLabel = {
84
- type: 'info',
85
- name: 'Syncing',
86
- spinner: true
87
- };
88
- break;
89
-
90
- case 'connected':
91
- account.stateLabel = {
92
- type: 'success',
93
- name: 'Connected'
94
- };
95
- break;
96
-
97
- case 'disabled':
98
- account.stateLabel = {
99
- type: 'secondary',
100
- name: 'Disabled',
101
- error: account.disabledReason
102
- };
103
- break;
104
-
105
- case 'authenticationError':
106
- case 'connectError': {
107
- let errorMessage = account.lastErrorState ? account.lastErrorState.response : false;
108
- if (account.lastErrorState) {
109
- switch (account.lastErrorState.serverResponseCode) {
110
- case 'ETIMEDOUT':
111
- errorMessage = gt.gettext('Connection timed out. This usually occurs if you are behind a firewall or connecting to the wrong port.');
112
- break;
113
- case 'ClosedAfterConnectTLS':
114
- errorMessage = gt.gettext('The server unexpectedly closed the connection.');
115
- break;
116
- case 'ClosedAfterConnectText':
117
- errorMessage = gt.gettext(
118
- 'The server unexpectedly closed the connection. This usually happens when attempting to connect to a TLS port without TLS enabled.'
119
- );
120
- break;
121
- case 'ECONNREFUSED':
122
- errorMessage = gt.gettext(
123
- 'The server refused the connection. This typically occurs if the server is not running, is overloaded, or you are connecting to the wrong host or port.'
124
- );
125
- break;
126
- }
127
- }
128
-
129
- account.stateLabel = {
130
- type: 'danger',
131
- name: 'Failed',
132
- error: errorMessage
133
- };
134
- break;
135
- }
136
- case 'unset':
137
- account.stateLabel = {
138
- type: 'light',
139
- name: 'Not syncing'
140
- };
141
- break;
142
- case 'disconnected':
143
- account.stateLabel = {
144
- type: 'warning',
145
- name: 'Disconnected'
146
- };
147
- break;
148
- case 'paused':
149
- account.stateLabel = {
150
- type: 'secondary',
151
- name: 'Paused'
152
- };
153
- break;
154
- default:
155
- account.stateLabel = {
156
- type: 'secondary',
157
- name: 'N/A'
158
- };
159
- break;
160
- }
161
-
162
- // Check if IMAP was disabled due to errors - override state label to show error
163
- if (account.imap && account.imap.disabled && account.lastErrorState) {
164
- account.stateLabel = {
165
- type: 'danger',
166
- name: 'Failed',
167
- error: account.lastErrorState.description || account.lastErrorState.response
168
- };
169
- }
170
-
171
- if (account.oauth2) {
172
- account.oauth2.scopes = []
173
- .concat(account.oauth2.scope || [])
174
- .concat(account.oauth2.scopes || [])
175
- .flatMap(entry => entry.split(/\s+/))
176
- .map(entry => entry.trim())
177
- .filter(entry => entry);
178
-
179
- account.oauth2.expiresStr = account.oauth2.expires ? account.oauth2.expires.toISOString() : false;
180
- account.oauth2.generatedStr = account.oauth2.generated ? account.oauth2.generated.toISOString() : false;
181
-
182
- if (account.outlookSubscription) {
183
- account.outlookSubscription.subscriptionExpiresStr = account.outlookSubscription.expirationDateTime
184
- ? account.outlookSubscription.expirationDateTime.toISOString()
185
- : false;
186
-
187
- let state = account.outlookSubscription.state || {};
188
-
189
- account.outlookSubscription.isValid =
190
- state.state !== 'error' && account.outlookSubscription.expirationDateTime && account.outlookSubscription.expirationDateTime > new Date();
191
-
192
- account.outlookSubscription.stateLabel = (state.state || '').replace(/^./, c => c.toUpperCase());
193
-
194
- if ((state.state === 'created' && !account.outlookSubscription.expirationDateTime) || account.outlookSubscription.expirationDateTime < new Date()) {
195
- account.outlookSubscription.stateLabel = 'Expired';
196
- }
197
- }
198
- }
199
-
200
- return account;
201
- }
202
-
203
41
  async function getMailboxListing(accountObject) {
204
42
  let mailboxes = [
205
43
  {
@@ -229,7 +67,7 @@ async function getMailboxListing(accountObject) {
229
67
  return a.path.toLowerCase().localeCompare(b.path.toLowerCase());
230
68
  });
231
69
  } catch (err) {
232
- // ignore
70
+ // failed to get mailbox list
233
71
  }
234
72
 
235
73
  return mailboxes;
@@ -238,7 +76,6 @@ async function getMailboxListing(accountObject) {
238
76
  function init(args) {
239
77
  const { server, call } = args;
240
78
 
241
- // Account listing route
242
79
  server.route({
243
80
  method: 'GET',
244
81
  path: '/admin/accounts',
@@ -256,14 +93,21 @@ function init(args) {
256
93
  }
257
94
 
258
95
  for (let account of accounts.accounts) {
259
- let accountObj = new Account({ redis, account: account.account });
260
- account.data = await accountObj.loadAccountData(null, null, runIndex);
96
+ let accountObject = new Account({ redis, account: account.account });
97
+ try {
98
+ account.data = await accountObject.loadAccountData(null, null, runIndex);
261
99
 
262
- if (account.data && account.data.oauth2 && account.data.oauth2.provider) {
263
- let oauth2App = await oauth2Apps.get(account.data.oauth2.provider);
264
- if (oauth2App) {
265
- account.data.oauth2.app = oauth2App;
100
+ if (account.data && account.data.oauth2 && account.data.oauth2.provider) {
101
+ let oauth2App = await oauth2Apps.get(account.data.oauth2.provider);
102
+ if (oauth2App) {
103
+ account.data.oauth2.app = oauth2App;
104
+ }
266
105
  }
106
+ } catch (err) {
107
+ // Account has invalid config (e.g., broken delegation)
108
+ account.data = {
109
+ delegationError: err.message
110
+ };
267
111
  }
268
112
  }
269
113
 
@@ -333,6 +177,11 @@ function init(args) {
333
177
  label: 'Disconnected'
334
178
  },
335
179
 
180
+ {
181
+ state: 'paused',
182
+ label: 'Paused'
183
+ },
184
+
336
185
  {
337
186
  state: 'authenticationError',
338
187
  label: 'Authentication failed'
@@ -365,7 +214,7 @@ function init(args) {
365
214
  selectedState: stateOptions.find(entry => entry.state && entry.state === request.query.state),
366
215
 
367
216
  searchTarget: '/admin/accounts',
368
- searchPlaceholder: 'Search for accounts...',
217
+ searchPlaceholder: 'Search for accounts',
369
218
 
370
219
  showPaging: accounts.pages > 1,
371
220
  nextPage,
@@ -406,7 +255,7 @@ function init(args) {
406
255
  state: Joi.string()
407
256
  .trim()
408
257
  .empty('')
409
- .valid('init', 'syncing', 'connecting', 'connected', 'authenticationError', 'connectError', 'unset', 'disconnected')
258
+ .valid(...ACCOUNT_DISPLAY_STATES)
410
259
  .example('connected')
411
260
  .description('Filter accounts by state')
412
261
  .label('AccountState')
@@ -415,7 +264,6 @@ function init(args) {
415
264
  }
416
265
  });
417
266
 
418
- // New account POST handler
419
267
  server.route({
420
268
  method: 'POST',
421
269
  path: '/admin/accounts/new',
@@ -566,7 +414,6 @@ function init(args) {
566
414
  );
567
415
  }
568
416
 
569
- // Public GET new account form
570
417
  server.route({
571
418
  method: 'GET',
572
419
  path: '/accounts/new',
@@ -628,7 +475,6 @@ function init(args) {
628
475
  }
629
476
  });
630
477
 
631
- // Public POST new account form
632
478
  server.route({
633
479
  method: 'POST',
634
480
  path: '/accounts/new',
@@ -669,7 +515,6 @@ function init(args) {
669
515
  }
670
516
  });
671
517
 
672
- // IMAP account setup form
673
518
  server.route({
674
519
  method: 'POST',
675
520
  path: '/accounts/new/imap',
@@ -772,7 +617,6 @@ function init(args) {
772
617
  }
773
618
  });
774
619
 
775
- // Test IMAP settings
776
620
  server.route({
777
621
  method: 'POST',
778
622
  path: '/accounts/new/imap/test',
@@ -924,7 +768,6 @@ function init(args) {
924
768
  }
925
769
  });
926
770
 
927
- // Submit IMAP server settings
928
771
  server.route({
929
772
  method: 'POST',
930
773
  path: '/accounts/new/imap/server',
@@ -1097,7 +940,6 @@ function init(args) {
1097
940
  }
1098
941
  });
1099
942
 
1100
- // View account details
1101
943
  server.route({
1102
944
  method: 'GET',
1103
945
  path: '/admin/accounts/{account}',
@@ -1327,7 +1169,6 @@ function init(args) {
1327
1169
  }
1328
1170
  });
1329
1171
 
1330
- // Delete account
1331
1172
  server.route({
1332
1173
  method: 'POST',
1333
1174
  path: '/admin/accounts/{account}/delete',
@@ -1369,7 +1210,6 @@ function init(args) {
1369
1210
  }
1370
1211
  });
1371
1212
 
1372
- // Reconnect account
1373
1213
  server.route({
1374
1214
  method: 'POST',
1375
1215
  path: '/admin/accounts/{account}/reconnect',
@@ -1408,7 +1248,6 @@ function init(args) {
1408
1248
  }
1409
1249
  });
1410
1250
 
1411
- // Sync account
1412
1251
  server.route({
1413
1252
  method: 'POST',
1414
1253
  path: '/admin/accounts/{account}/sync',
@@ -1447,7 +1286,6 @@ function init(args) {
1447
1286
  }
1448
1287
  });
1449
1288
 
1450
- // Toggle account logs
1451
1289
  server.route({
1452
1290
  method: 'POST',
1453
1291
  path: '/admin/accounts/{account}/logs',
@@ -1495,7 +1333,6 @@ function init(args) {
1495
1333
  }
1496
1334
  });
1497
1335
 
1498
- // Flush account logs
1499
1336
  server.route({
1500
1337
  method: 'POST',
1501
1338
  path: '/admin/accounts/{account}/logs-flush',
@@ -1532,7 +1369,6 @@ function init(args) {
1532
1369
  }
1533
1370
  });
1534
1371
 
1535
- // Get account logs as text
1536
1372
  server.route({
1537
1373
  method: 'GET',
1538
1374
  path: '/admin/accounts/{account}/logs.txt',
@@ -1556,7 +1392,6 @@ function init(args) {
1556
1392
  }
1557
1393
  });
1558
1394
 
1559
- // Browse account messages
1560
1395
  server.route({
1561
1396
  method: 'GET',
1562
1397
  path: '/admin/accounts/{account}/browse',
@@ -1573,7 +1408,7 @@ function init(args) {
1573
1408
  if (request.cookieAuth) {
1574
1409
  request.cookieAuth.clear();
1575
1410
  }
1576
- await request.flash({ type: 'info', message: `Sign in again to continue` });
1411
+ await request.flash({ type: 'info', message: `Sign in again to continue.` });
1577
1412
  return h.redirect('/admin/login?next=' + encodeURIComponent('/admin/accounts/{account}/browse'));
1578
1413
  }
1579
1414
 
@@ -1635,7 +1470,6 @@ function init(args) {
1635
1470
  }
1636
1471
  });
1637
1472
 
1638
- // Edit account (GET)
1639
1473
  server.route({
1640
1474
  method: 'GET',
1641
1475
  path: '/admin/accounts/{account}/edit',
@@ -1715,7 +1549,6 @@ function init(args) {
1715
1549
  }
1716
1550
  });
1717
1551
 
1718
- // Edit account (POST)
1719
1552
  server.route({
1720
1553
  method: 'POST',
1721
1554
  path: '/admin/accounts/{account}/edit',