emailengine-app 2.68.1 → 2.70.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. package/.github/workflows/deploy.yml +8 -3
  2. package/.github/workflows/release.yaml +6 -0
  3. package/CHANGELOG.md +59 -0
  4. package/Gruntfile.js +3 -1
  5. package/config/default.toml +2 -0
  6. package/data/google-crawlers.json +7 -1
  7. package/getswagger.sh +40 -4
  8. package/gettext-extract.js +163 -0
  9. package/lib/account.js +135 -72
  10. package/lib/api-routes/account-routes.js +684 -106
  11. package/lib/api-routes/blocklist-routes.js +344 -0
  12. package/lib/api-routes/chat-routes.js +32 -14
  13. package/lib/api-routes/delivery-test-routes.js +346 -0
  14. package/lib/api-routes/export-routes.js +28 -14
  15. package/lib/api-routes/gateway-routes.js +427 -0
  16. package/lib/api-routes/license-routes.js +156 -0
  17. package/lib/api-routes/mailbox-routes.js +344 -0
  18. package/lib/api-routes/message-routes.js +221 -187
  19. package/lib/api-routes/oauth2-app-routes.js +697 -0
  20. package/lib/api-routes/outbox-routes.js +185 -0
  21. package/lib/api-routes/pubsub-routes.js +102 -0
  22. package/lib/api-routes/route-helpers.js +58 -0
  23. package/lib/api-routes/settings-routes.js +357 -0
  24. package/lib/api-routes/stats-routes.js +111 -0
  25. package/lib/api-routes/submit-routes.js +461 -0
  26. package/lib/api-routes/template-routes.js +60 -75
  27. package/lib/api-routes/token-routes.js +297 -0
  28. package/lib/api-routes/webhook-route-routes.js +181 -0
  29. package/lib/autodetect-imap-settings.js +0 -2
  30. package/lib/consts.js +5 -0
  31. package/lib/email-client/base-client.js +28 -6
  32. package/lib/email-client/gmail-client.js +133 -112
  33. package/lib/email-client/imap/mailbox.js +34 -11
  34. package/lib/email-client/imap/subconnection.js +20 -13
  35. package/lib/email-client/imap/sync-operations.js +131 -3
  36. package/lib/email-client/imap-client.js +152 -75
  37. package/lib/email-client/notification-handler.js +1 -4
  38. package/lib/email-client/outlook-client.js +134 -75
  39. package/lib/export.js +97 -20
  40. package/lib/feature-flags.js +2 -2
  41. package/lib/gateway.js +4 -9
  42. package/lib/get-raw-email.js +5 -5
  43. package/lib/imapproxy/imap-core/lib/commands/starttls.js +18 -0
  44. package/lib/imapproxy/imap-core/lib/imap-command.js +6 -1
  45. package/lib/imapproxy/imap-core/lib/imap-connection.js +106 -24
  46. package/lib/imapproxy/imap-core/lib/imap-server.js +24 -0
  47. package/lib/imapproxy/imap-core/lib/imap-stream.js +26 -0
  48. package/lib/logger.js +24 -21
  49. package/lib/message-port-stream.js +113 -16
  50. package/lib/metrics-collector.js +0 -2
  51. package/lib/oauth2-apps.js +13 -4
  52. package/lib/outbox.js +24 -40
  53. package/lib/redis-operations.js +1 -1
  54. package/lib/reject-worker-calls.js +42 -0
  55. package/lib/routes-ui.js +37 -8778
  56. package/lib/schemas.js +429 -84
  57. package/lib/sentry.js +139 -0
  58. package/lib/settings.js +9 -3
  59. package/lib/stream-encrypt.js +1 -1
  60. package/lib/templates.js +1 -1
  61. package/lib/tokens.js +5 -3
  62. package/lib/tools.js +70 -4
  63. package/lib/ui-routes/account-routes.js +45 -212
  64. package/lib/ui-routes/admin-config-routes.js +928 -489
  65. package/lib/ui-routes/admin-entities-routes.js +1 -0
  66. package/lib/ui-routes/auth-routes.js +1339 -0
  67. package/lib/ui-routes/dashboard-routes.js +188 -0
  68. package/lib/ui-routes/document-store-routes.js +800 -0
  69. package/lib/ui-routes/export-routes.js +217 -0
  70. package/lib/ui-routes/internals-routes.js +354 -0
  71. package/lib/ui-routes/network-config-routes.js +759 -0
  72. package/lib/ui-routes/{oauth-routes.js → oauth-config-routes.js} +369 -91
  73. package/lib/ui-routes/route-helpers.js +314 -0
  74. package/lib/ui-routes/smtp-test-routes.js +236 -0
  75. package/lib/ui-routes/unsubscribe-routes.js +232 -0
  76. package/lib/webhook-request.js +36 -0
  77. package/lib/webhooks.js +8 -4
  78. package/package.json +13 -12
  79. package/sbom.json +1 -1
  80. package/server.js +222 -39
  81. package/static/licenses.html +160 -300
  82. package/translations/messages.pot +112 -132
  83. package/update-info.sh +19 -1
  84. package/views/config/logging.hbs +48 -0
  85. package/views/dashboard.hbs +7 -26
  86. package/views/internals/index.hbs +15 -0
  87. package/views/tokens/index.hbs +9 -0
  88. package/workers/api.js +200 -4424
  89. package/workers/documents.js +2 -22
  90. package/workers/export.js +103 -104
  91. package/workers/imap-proxy.js +3 -23
  92. package/workers/imap.js +32 -36
  93. package/workers/smtp.js +2 -22
  94. package/workers/submit.js +26 -35
  95. package/workers/webhooks.js +9 -43
@@ -14,28 +14,8 @@ const GB_COLLECT_DELAY = 6 * 3600 * 1000; // 6h
14
14
  const GB_FAILURE_DELAY = 3 * 1000;
15
15
  const GB_EMPTY_DELAY = 10 * 1000;
16
16
 
17
- const Bugsnag = require('@bugsnag/js');
18
- if (readEnvValue('BUGSNAG_API_KEY')) {
19
- Bugsnag.start({
20
- apiKey: readEnvValue('BUGSNAG_API_KEY'),
21
- appVersion: packageData.version,
22
- logger: {
23
- debug(...args) {
24
- logger.debug({ msg: args.shift(), worker: 'documents', source: 'bugsnag', args: args.length ? args : undefined });
25
- },
26
- info(...args) {
27
- logger.debug({ msg: args.shift(), worker: 'documents', source: 'bugsnag', args: args.length ? args : undefined });
28
- },
29
- warn(...args) {
30
- logger.warn({ msg: args.shift(), worker: 'documents', source: 'bugsnag', args: args.length ? args : undefined });
31
- },
32
- error(...args) {
33
- logger.error({ msg: args.shift(), worker: 'documents', source: 'bugsnag', args: args.length ? args : undefined });
34
- }
35
- }
36
- });
37
- logger.notifyError = Bugsnag.notify.bind(Bugsnag);
38
- }
17
+ const { initSentry } = require('../lib/sentry');
18
+ initSentry('documents');
39
19
 
40
20
  const { redis, queueConf } = require('../lib/db');
41
21
  const { Worker } = require('bullmq');
package/workers/export.js CHANGED
@@ -17,33 +17,13 @@ const {
17
17
  DEFAULT_EXPORT_MAX_MESSAGES,
18
18
  DEFAULT_EXPORT_MAX_SIZE
19
19
  } = require('../lib/consts');
20
- const { getDuration, readEnvValue, threadStats, reloadHttpProxyAgent } = require('../lib/tools');
20
+ const { getDuration, readEnvValue, threadStats, maybeReloadHttpProxyAgent } = require('../lib/tools');
21
21
  const { webhooks: Webhooks } = require('../lib/webhooks');
22
22
  const settings = require('../lib/settings');
23
- const { Export } = require('../lib/export');
24
-
25
- const Bugsnag = require('@bugsnag/js');
26
- if (readEnvValue('BUGSNAG_API_KEY')) {
27
- Bugsnag.start({
28
- apiKey: readEnvValue('BUGSNAG_API_KEY'),
29
- appVersion: packageData.version,
30
- logger: {
31
- debug(...args) {
32
- logger.debug({ msg: args.shift(), worker: 'export', source: 'bugsnag', args: args.length ? args : undefined });
33
- },
34
- info(...args) {
35
- logger.debug({ msg: args.shift(), worker: 'export', source: 'bugsnag', args: args.length ? args : undefined });
36
- },
37
- warn(...args) {
38
- logger.warn({ msg: args.shift(), worker: 'export', source: 'bugsnag', args: args.length ? args : undefined });
39
- },
40
- error(...args) {
41
- logger.error({ msg: args.shift(), worker: 'export', source: 'bugsnag', args: args.length ? args : undefined });
42
- }
43
- }
44
- });
45
- logger.notifyError = Bugsnag.notify.bind(Bugsnag);
46
- }
23
+ const { Export, isTransientError, isSkippableError, isFolderMissingError } = require('../lib/export');
24
+
25
+ const { initSentry } = require('../lib/sentry');
26
+ initSentry('export');
47
27
 
48
28
  const { redis, queueConf } = require('../lib/db');
49
29
  const { Worker } = require('bullmq');
@@ -74,23 +54,6 @@ const IMAP_MESSAGE_RETRY_BASE_DELAY = 2000;
74
54
  const ACCOUNT_CHECK_INTERVAL = 60 * 1000;
75
55
  const LOCK_EXTENSION_INTERVAL = 5 * 60 * 1000;
76
56
 
77
- function isTransientError(err) {
78
- if (['ETIMEDOUT', 'ECONNRESET', 'ENOTFOUND', 'EAI_AGAIN', 'ECONNREFUSED', 'EPIPE', 'EHOSTUNREACH'].includes(err.code)) {
79
- return true;
80
- }
81
- if (err.statusCode >= 500 && err.statusCode < 600) {
82
- return true;
83
- }
84
- if (err.code === 'Timeout' || err.message?.includes('timeout')) {
85
- return true;
86
- }
87
- return false;
88
- }
89
-
90
- function isSkippableError(err) {
91
- return err.code === 'MessageNotFound' || err.statusCode === 404 || err.message?.includes('Failed to generate message ID');
92
- }
93
-
94
57
  let callQueue = new Map();
95
58
  let mids = 0;
96
59
 
@@ -155,6 +118,28 @@ async function notify(account, event, data) {
155
118
  await Webhooks.pushToQueue(event, await Webhooks.formatPayload(event, payload));
156
119
  }
157
120
 
121
+ // Returns a throttled callback that extends both the BullMQ job lock and the export's Redis key
122
+ // expiry on the LOCK_EXTENSION_INTERVAL cadence. Long-running indexing/exporting loops call it
123
+ // freely; the lock/expiry are refreshed at most once per interval. Without this the lock could lapse
124
+ // (stalling and reprocessing the job) and the export keys could expire mid-run.
125
+ function createLeaseExtender(job, account, exportId) {
126
+ let lastExtension = Date.now();
127
+ return async function maybeExtendLease() {
128
+ if (Date.now() - lastExtension <= LOCK_EXTENSION_INTERVAL) {
129
+ return;
130
+ }
131
+ await Promise.all([
132
+ job.extendLock(job.token, 10 * 60 * 1000).catch(err => {
133
+ logger.warn({ msg: 'Failed to extend job lock', account, exportId, err });
134
+ }),
135
+ Export.extendExpiry(account, exportId).catch(err => {
136
+ logger.warn({ msg: 'Failed to extend export expiry', account, exportId, err });
137
+ })
138
+ ]);
139
+ lastExtension = Date.now();
140
+ };
141
+ }
142
+
158
143
  async function indexMessages(job, exportData) {
159
144
  const { account, exportId } = job.data;
160
145
  const folders = JSON.parse(exportData.folders || '[]');
@@ -195,15 +180,10 @@ async function indexMessages(job, exportData) {
195
180
 
196
181
  let totalIndexed = 0;
197
182
  let truncated = false;
198
- let lastLockExtension = Date.now();
183
+ const maybeExtendLease = createLeaseExtender(job, account, exportId);
199
184
 
200
185
  for (let i = 0; i < foldersToProcess.length; i++) {
201
- if (Date.now() - lastLockExtension > LOCK_EXTENSION_INTERVAL) {
202
- await job.extendLock(job.token, 10 * 60 * 1000).catch(err => {
203
- logger.warn({ msg: 'Failed to extend job lock during indexing', account, exportId, err });
204
- });
205
- lastLockExtension = Date.now();
206
- }
186
+ await maybeExtendLease();
207
187
 
208
188
  if (await Export.isCancelled(account, exportId)) {
209
189
  const err = new Error('Export cancelled by user');
@@ -221,8 +201,6 @@ async function indexMessages(job, exportData) {
221
201
  const queued = await indexFolder(accountObject, account, exportId, folderPath, startDate, endDate, indexingStartTime, remaining);
222
202
  totalIndexed += queued;
223
203
 
224
- await Export.update(account, exportId, { foldersScanned: i + 1 });
225
-
226
204
  logger.trace({
227
205
  msg: 'Folder indexed',
228
206
  account,
@@ -266,6 +244,9 @@ async function indexMessages(job, exportData) {
266
244
  });
267
245
  }
268
246
 
247
+ // Count the folder as scanned whether or not it succeeded so progress reaches foldersTotal.
248
+ await Export.update(account, exportId, { foldersScanned: i + 1 });
249
+
269
250
  if (maxMessages && totalIndexed >= maxMessages) {
270
251
  truncated = true;
271
252
  logger.warn({
@@ -323,7 +304,18 @@ async function indexFolder(accountObject, account, exportId, folderPath, startDa
323
304
  cursor
324
305
  };
325
306
 
326
- const result = await accountObject.listMessages(listOptions);
307
+ let result;
308
+ try {
309
+ result = await accountObject.listMessages(listOptions);
310
+ } catch (err) {
311
+ // Other errors (e.g. a deleted account) propagate
312
+ if (isFolderMissingError(err)) {
313
+ // The folder disappeared mid-export - treat it as empty instead of failing the export
314
+ logger.warn({ msg: 'Export folder was not found, skipping', account, exportId, folder: folderPath, err });
315
+ return queued;
316
+ }
317
+ throw err;
318
+ }
327
319
 
328
320
  for (const msg of result.messages || []) {
329
321
  if (maxMessages && queued >= maxMessages) {
@@ -357,7 +349,9 @@ async function exportMessages(job, exportData) {
357
349
  const { filePath } = exportData;
358
350
  const includeAttachments = exportData.includeAttachments === '1';
359
351
  const textType = exportData.textType || '*';
360
- const maxBytes = Number(exportData.maxBytes) || 5 * 1024 * 1024;
352
+ const rawMaxBytes = Number(exportData.maxBytes);
353
+ // Preserve an explicit 0 ("unlimited" per the API contract); only fall back when unset/invalid.
354
+ const maxBytes = Number.isFinite(rawMaxBytes) ? rawMaxBytes : 5 * 1024 * 1024;
361
355
  const maxMessageSize = (await settings.get('exportMaxMessageSize')) || DEFAULT_EXPORT_MAX_MESSAGE_SIZE;
362
356
  const maxExportSize = Number(await settings.get('exportMaxSize')) || DEFAULT_EXPORT_MAX_SIZE;
363
357
  const isEncrypted = exportData.isEncrypted === '1';
@@ -419,14 +413,15 @@ async function exportMessages(job, exportData) {
419
413
  });
420
414
  }
421
415
 
422
- let lastScore = Number(exportData.lastProcessedScore) || 0;
423
416
  let processed = 0;
424
417
  let totalBytesWritten = 0;
425
418
  let processingError = null;
426
419
  let sizeLimitReached = false;
427
420
 
428
421
  let lastAccountCheck = Date.now();
429
- let lastLockExtension = Date.now();
422
+ // Refreshes the job lock and export key expiry on a fixed cadence; called from the main loop and
423
+ // from inside the rate-limit backoff so a batch stuck retrying does not let the lock lapse.
424
+ const maybeExtendLease = createLeaseExtender(job, account, exportId);
430
425
 
431
426
  const accountData = await accountObject.loadAccountData(account);
432
427
  const isApiAccount = await accountObject.isApiClient(accountData);
@@ -490,12 +485,7 @@ async function exportMessages(job, exportData) {
490
485
  break;
491
486
  }
492
487
 
493
- if (Date.now() - lastLockExtension > LOCK_EXTENSION_INTERVAL) {
494
- await job.extendLock(job.token, 10 * 60 * 1000).catch(err => {
495
- logger.warn({ msg: 'Failed to extend job lock', account, exportId, err });
496
- });
497
- lastLockExtension = Date.now();
498
- }
488
+ await maybeExtendLease();
499
489
 
500
490
  if (await Export.isCancelled(account, exportId)) {
501
491
  const err = new Error('Export cancelled by user');
@@ -513,7 +503,7 @@ async function exportMessages(job, exportData) {
513
503
  lastAccountCheck = Date.now();
514
504
  }
515
505
 
516
- const batch = await Export.getNextBatch(account, exportId, lastScore, BATCH_SIZE);
506
+ const batch = await Export.getNextBatch(account, exportId, BATCH_SIZE);
517
507
  if (batch.length === 0) {
518
508
  break;
519
509
  }
@@ -522,14 +512,12 @@ async function exportMessages(job, exportData) {
522
512
  for (const entry of batch) {
523
513
  if (includeAttachments && entry.size > maxMessageSize) {
524
514
  await Export.incrementSkipped(account, exportId);
525
- lastScore = entry.score;
526
515
  } else {
527
516
  entriesToFetch.push(entry);
528
517
  }
529
518
  }
530
519
 
531
520
  if (entriesToFetch.length === 0) {
532
- await Export.updateLastProcessedScore(account, exportId, lastScore);
533
521
  continue;
534
522
  }
535
523
 
@@ -573,7 +561,6 @@ async function exportMessages(job, exportData) {
573
561
  reason: err.message || err.code
574
562
  });
575
563
  await Export.incrementSkipped(account, exportId);
576
- lastScore = entry.score;
577
564
  } else if (isRateLimited && rateLimitRetry < MAX_RATE_LIMIT_RETRIES) {
578
565
  rateLimitedEntries.push(entry);
579
566
  } else {
@@ -584,7 +571,6 @@ async function exportMessages(job, exportData) {
584
571
  }
585
572
  } else if (result && result.data) {
586
573
  await processMessage(result.data, entry);
587
- lastScore = entry.score;
588
574
  if (maxExportSize && totalBytesWritten >= maxExportSize) {
589
575
  sizeLimitReached = true;
590
576
  break;
@@ -599,7 +585,6 @@ async function exportMessages(job, exportData) {
599
585
  reason: 'Message not found in batch results'
600
586
  });
601
587
  await Export.incrementSkipped(account, exportId);
602
- lastScore = entry.score;
603
588
  }
604
589
  }
605
590
 
@@ -620,6 +605,7 @@ async function exportMessages(job, exportData) {
620
605
  delayMs: Math.round(delay)
621
606
  });
622
607
  await new Promise(resolve => setTimeout(resolve, delay));
608
+ await maybeExtendLease();
623
609
  fetchBatch = rateLimitedEntries;
624
610
  } else {
625
611
  break;
@@ -672,7 +658,6 @@ async function exportMessages(job, exportData) {
672
658
  await processMessage(message, entry);
673
659
  if (maxExportSize && totalBytesWritten >= maxExportSize) {
674
660
  sizeLimitReached = true;
675
- lastScore = entry.score;
676
661
  break;
677
662
  }
678
663
  } else if (fetchError && isSkippableError(fetchError)) {
@@ -688,12 +673,9 @@ async function exportMessages(job, exportData) {
688
673
  } else if (fetchError) {
689
674
  throw fetchError;
690
675
  }
691
-
692
- lastScore = entry.score;
693
676
  }
694
677
  }
695
678
 
696
- await Export.updateLastProcessedScore(account, exportId, lastScore);
697
679
  logger.trace({ msg: 'Export batch processed', account, exportId, messagesExported: processed });
698
680
  }
699
681
  } catch (err) {
@@ -764,19 +746,26 @@ const exportWorker = new Worker(
764
746
 
765
747
  await Export.complete(account, exportId);
766
748
 
767
- const finalData = await redis.hgetall(`${REDIS_PREFIX}exp:${account}:${exportId}`);
749
+ // The export is complete and persisted. A failure while reading final stats or delivering
750
+ // the completion webhook must NOT fall through to the job catch below, which would delete
751
+ // the finished file and mark the export as failed.
752
+ try {
753
+ const finalData = await redis.hgetall(`${REDIS_PREFIX}exp:${account}:${exportId}`);
768
754
 
769
- await notify(account, EXPORT_COMPLETED_NOTIFY, {
770
- exportId,
771
- folders: JSON.parse(exportData.folders || '[]'),
772
- startDate: new Date(Number(exportData.startDate)).toISOString(),
773
- endDate: new Date(Number(exportData.endDate)).toISOString(),
774
- messagesExported: Number(finalData.messagesExported) || 0,
775
- messagesSkipped: Number(finalData.messagesSkipped) || 0,
776
- bytesWritten: Number(finalData.bytesWritten) || 0,
777
- duration: Date.now() - startTime,
778
- expiresAt: new Date(Number(finalData.expiresAt)).toISOString()
779
- });
755
+ await notify(account, EXPORT_COMPLETED_NOTIFY, {
756
+ exportId,
757
+ folders: JSON.parse(exportData.folders || '[]'),
758
+ startDate: new Date(Number(exportData.startDate)).toISOString(),
759
+ endDate: new Date(Number(exportData.endDate)).toISOString(),
760
+ messagesExported: Number(finalData.messagesExported) || 0,
761
+ messagesSkipped: Number(finalData.messagesSkipped) || 0,
762
+ bytesWritten: Number(finalData.bytesWritten) || 0,
763
+ duration: Date.now() - startTime,
764
+ expiresAt: new Date(Number(finalData.expiresAt)).toISOString()
765
+ });
766
+ } catch (notifyErr) {
767
+ logger.error({ msg: 'Failed to deliver export completion notification', account, exportId, err: notifyErr });
768
+ }
780
769
 
781
770
  logger.info({ msg: 'Export job completed', account, exportId, duration: Date.now() - startTime });
782
771
  } catch (err) {
@@ -795,14 +784,19 @@ const exportWorker = new Worker(
795
784
  }
796
785
 
797
786
  if (err.code !== 'AccountDeleted' && err.code !== 'AccountNotFound' && err.code !== 'ExportCancelled') {
798
- await notify(account, EXPORT_FAILED_NOTIFY, {
799
- exportId,
800
- error: err.message,
801
- errorCode: err.code,
802
- phase: exportData.phase || 'unknown',
803
- messagesExported: Number(exportData.messagesExported) || 0,
804
- messagesQueued: Number(exportData.messagesQueued) || 0
805
- });
787
+ try {
788
+ await notify(account, EXPORT_FAILED_NOTIFY, {
789
+ exportId,
790
+ error: err.message,
791
+ errorCode: err.code,
792
+ phase: exportData.phase || 'unknown',
793
+ messagesExported: Number(exportData.messagesExported) || 0,
794
+ messagesQueued: Number(exportData.messagesQueued) || 0
795
+ });
796
+ } catch (notifyErr) {
797
+ // Never let a failed notification replace the original error below.
798
+ logger.error({ msg: 'Failed to deliver export failure notification', account, exportId, err: notifyErr });
799
+ }
806
800
  }
807
801
 
808
802
  throw err;
@@ -876,7 +870,7 @@ function onCommand(command) {
876
870
  try {
877
871
  const activeExports = await redis.smembers(`${REDIS_PREFIX}exp:active`);
878
872
  for (const entry of activeExports) {
879
- const separatorIndex = entry.indexOf(':exp_');
873
+ const separatorIndex = entry.lastIndexOf(':exp_');
880
874
  if (separatorIndex === -1) continue;
881
875
  const entryAccount = entry.substring(0, separatorIndex);
882
876
  const entryExportId = entry.substring(separatorIndex + 1);
@@ -893,6 +887,22 @@ function onCommand(command) {
893
887
  5 * 60 * 1000
894
888
  ).unref();
895
889
 
890
+ // Periodically remove export files whose Redis state has already expired. Without this, files of
891
+ // expired exports would only be reclaimed on the next worker restart.
892
+ setInterval(
893
+ async () => {
894
+ try {
895
+ const cleaned = await Export.cleanup();
896
+ if (cleaned > 0) {
897
+ logger.info({ msg: 'Cleaned up orphaned export files', count: cleaned });
898
+ }
899
+ } catch (err) {
900
+ logger.error({ msg: 'Failed to clean up export files', err });
901
+ }
902
+ },
903
+ 60 * 60 * 1000
904
+ ).unref();
905
+
896
906
  parentPort.postMessage({ cmd: 'ready' });
897
907
  })();
898
908
 
@@ -919,19 +929,8 @@ parentPort.on('message', message => {
919
929
  }
920
930
 
921
931
  if (message && message.cmd === 'settings') {
922
- let d = message.data || {};
923
- if ('httpProxyEnabled' in d || 'httpProxyUrl' in d) {
924
- reloadHttpProxyAgent().catch(err => logger.error({ msg: 'Failed to reload HTTP proxy agent', err }));
925
- }
932
+ maybeReloadHttpProxyAgent(message.data);
926
933
  }
927
934
  });
928
935
 
929
936
  logger.info({ msg: 'Started export worker thread', version: packageData.version });
930
-
931
- module.exports = {
932
- isTransientError,
933
- isSkippableError,
934
- IMAP_MESSAGE_MAX_RETRIES,
935
- IMAP_MESSAGE_RETRY_BASE_DELAY,
936
- ACCOUNT_CHECK_INTERVAL
937
- };
@@ -5,32 +5,12 @@ const { parentPort } = require('worker_threads');
5
5
  const packageData = require('../package.json');
6
6
  const logger = require('../lib/logger');
7
7
 
8
- const { readEnvValue, threadStats } = require('../lib/tools');
8
+ const { threadStats } = require('../lib/tools');
9
9
 
10
10
  const { run } = require('../lib/imapproxy/imap-server');
11
11
 
12
- const Bugsnag = require('@bugsnag/js');
13
- if (readEnvValue('BUGSNAG_API_KEY')) {
14
- Bugsnag.start({
15
- apiKey: readEnvValue('BUGSNAG_API_KEY'),
16
- appVersion: packageData.version,
17
- logger: {
18
- debug(...args) {
19
- logger.debug({ msg: args.shift(), worker: 'imapProxy', source: 'bugsnag', args: args.length ? args : undefined });
20
- },
21
- info(...args) {
22
- logger.debug({ msg: args.shift(), worker: 'imapProxy', source: 'bugsnag', args: args.length ? args : undefined });
23
- },
24
- warn(...args) {
25
- logger.warn({ msg: args.shift(), worker: 'imapProxy', source: 'bugsnag', args: args.length ? args : undefined });
26
- },
27
- error(...args) {
28
- logger.error({ msg: args.shift(), worker: 'imapProxy', source: 'bugsnag', args: args.length ? args : undefined });
29
- }
30
- }
31
- });
32
- logger.notifyError = Bugsnag.notify.bind(Bugsnag);
33
- }
12
+ const { initSentry } = require('../lib/sentry');
13
+ initSentry('imapProxy');
34
14
 
35
15
  async function onCommand(command) {
36
16
  switch (command.cmd) {
package/workers/imap.js CHANGED
@@ -7,30 +7,10 @@ const logger = require('../lib/logger');
7
7
 
8
8
  const { REDIS_PREFIX } = require('../lib/consts');
9
9
 
10
- const { getDuration, getBoolean, emitChangeEvent, readEnvValue, hasEnvValue, threadStats, reloadHttpProxyAgent } = require('../lib/tools');
11
-
12
- const Bugsnag = require('@bugsnag/js');
13
- if (readEnvValue('BUGSNAG_API_KEY')) {
14
- Bugsnag.start({
15
- apiKey: readEnvValue('BUGSNAG_API_KEY'),
16
- appVersion: packageData.version,
17
- logger: {
18
- debug(...args) {
19
- logger.debug({ msg: args.shift(), worker: 'imap', source: 'bugsnag', args: args.length ? args : undefined });
20
- },
21
- info(...args) {
22
- logger.debug({ msg: args.shift(), worker: 'imap', source: 'bugsnag', args: args.length ? args : undefined });
23
- },
24
- warn(...args) {
25
- logger.warn({ msg: args.shift(), worker: 'imap', source: 'bugsnag', args: args.length ? args : undefined });
26
- },
27
- error(...args) {
28
- logger.error({ msg: args.shift(), worker: 'imap', source: 'bugsnag', args: args.length ? args : undefined });
29
- }
30
- }
31
- });
32
- logger.notifyError = Bugsnag.notify.bind(Bugsnag);
33
- }
10
+ const { getDuration, getBoolean, emitChangeEvent, readEnvValue, hasEnvValue, threadStats, maybeReloadHttpProxyAgent } = require('../lib/tools');
11
+
12
+ const { initSentry } = require('../lib/sentry');
13
+ initSentry('imap');
34
14
 
35
15
  const { IMAPClient } = require('../lib/email-client/imap-client');
36
16
  const { GmailClient } = require('../lib/email-client/gmail-client');
@@ -39,7 +19,7 @@ const { BaseClient } = require('../lib/email-client/base-client');
39
19
  const { Account } = require('../lib/account');
40
20
  const { oauth2Apps, isApiBasedApp } = require('../lib/oauth2-apps');
41
21
  const { redis, notifyQueue, submitQueue, documentsQueue, getFlowProducer } = require('../lib/db');
42
- const { MessagePortWritable } = require('../lib/message-port-stream');
22
+ const { MessagePortWritable, pipeToMessagePort } = require('../lib/message-port-stream');
43
23
  const { getESClient } = require('../lib/document-store');
44
24
  const settings = require('../lib/settings');
45
25
  const msgpack = require('msgpack5')();
@@ -728,8 +708,6 @@ class ConnectionHandler {
728
708
  if (!accountData.connection) {
729
709
  throw NO_ACTIVE_HANDLER_RESP_ERR;
730
710
  }
731
- let stream = new MessagePortWritable(message.port);
732
-
733
711
  let source = await accountData.connection.getRawMessage(message.message);
734
712
  if (!source) {
735
713
  let err = new Error('Requested file not found');
@@ -737,11 +715,22 @@ class ConnectionHandler {
737
715
  throw err;
738
716
  }
739
717
 
718
+ // Build the cross-thread writable only after we know there is content to send.
719
+ // Constructing it earlier leaks the transferred MessagePort (and its message
720
+ // listener) whenever the lookup above throws (e.g. a 404).
721
+ let stream = new MessagePortWritable(message.port);
722
+
740
723
  setImmediate(() => {
741
724
  if (Buffer.isBuffer(source)) {
742
- stream.end(source);
725
+ // The consumer may have aborted and destroyed the writable (via a cancel
726
+ // message) before this runs; ending a destroyed stream would emit an
727
+ // unhandled 'error' and take down the worker. The streaming branch is
728
+ // safe because pipeline() tears down a destroyed destination cleanly.
729
+ if (!stream.destroyed) {
730
+ stream.end(source);
731
+ }
743
732
  } else {
744
- source.pipe(stream);
733
+ pipeToMessagePort(source, stream, accountData.connection.logger);
745
734
  }
746
735
  });
747
736
 
@@ -764,8 +753,6 @@ class ConnectionHandler {
764
753
  throw NO_ACTIVE_HANDLER_RESP_ERR;
765
754
  }
766
755
 
767
- let stream = new MessagePortWritable(message.port);
768
-
769
756
  let source = await accountData.connection.getAttachment(message.attachment);
770
757
  if (!source) {
771
758
  let err = new Error('Requested file not found');
@@ -773,11 +760,22 @@ class ConnectionHandler {
773
760
  throw err;
774
761
  }
775
762
 
763
+ // Build the cross-thread writable only after we know there is content to send.
764
+ // Constructing it earlier leaks the transferred MessagePort (and its message
765
+ // listener) whenever the lookup above throws (e.g. a 404).
766
+ let stream = new MessagePortWritable(message.port);
767
+
776
768
  setImmediate(() => {
777
769
  if (Buffer.isBuffer(source.data)) {
778
- stream.end(source.data);
770
+ // The consumer may have aborted and destroyed the writable (via a cancel
771
+ // message) before this runs; ending a destroyed stream would emit an
772
+ // unhandled 'error' and take down the worker. The streaming branch is
773
+ // safe because pipeline() tears down a destroyed destination cleanly.
774
+ if (!stream.destroyed) {
775
+ stream.end(source.data);
776
+ }
779
777
  } else {
780
- source.pipe(stream);
778
+ pipeToMessagePort(source, stream, accountData.connection.logger);
781
779
  }
782
780
  });
783
781
 
@@ -812,9 +810,7 @@ class ConnectionHandler {
812
810
 
813
811
  switch (message.cmd) {
814
812
  case 'settings':
815
- if (message.data && ('httpProxyEnabled' in message.data || 'httpProxyUrl' in message.data)) {
816
- reloadHttpProxyAgent().catch(err => logger.error({ msg: 'Failed to reload HTTP proxy agent', err }));
817
- }
813
+ maybeReloadHttpProxyAgent(message.data);
818
814
 
819
815
  if (message.data && message.data.logs) {
820
816
  for (let [account, accountObject] of this.accounts) {
package/workers/smtp.js CHANGED
@@ -9,28 +9,8 @@ const logger = require('../lib/logger');
9
9
  const { getDuration, emitChangeEvent, readEnvValue, threadStats, loadTlsConfig, getByteSize } = require('../lib/tools');
10
10
  const { matchIp } = require('../lib/utils/network');
11
11
 
12
- const Bugsnag = require('@bugsnag/js');
13
- if (readEnvValue('BUGSNAG_API_KEY')) {
14
- Bugsnag.start({
15
- apiKey: readEnvValue('BUGSNAG_API_KEY'),
16
- appVersion: packageData.version,
17
- logger: {
18
- debug(...args) {
19
- logger.debug({ msg: args.shift(), worker: 'smtp', source: 'bugsnag', args: args.length ? args : undefined });
20
- },
21
- info(...args) {
22
- logger.debug({ msg: args.shift(), worker: 'smtp', source: 'bugsnag', args: args.length ? args : undefined });
23
- },
24
- warn(...args) {
25
- logger.warn({ msg: args.shift(), worker: 'smtp', source: 'bugsnag', args: args.length ? args : undefined });
26
- },
27
- error(...args) {
28
- logger.error({ msg: args.shift(), worker: 'smtp', source: 'bugsnag', args: args.length ? args : undefined });
29
- }
30
- }
31
- });
32
- logger.notifyError = Bugsnag.notify.bind(Bugsnag);
33
- }
12
+ const { initSentry } = require('../lib/sentry');
13
+ initSentry('smtp');
34
14
 
35
15
  const { SMTPServer } = require('smtp-server');
36
16
  const util = require('util');