emailengine-app 2.68.1 → 2.70.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/deploy.yml +8 -3
- package/.github/workflows/release.yaml +6 -0
- package/CHANGELOG.md +59 -0
- package/Gruntfile.js +3 -1
- package/config/default.toml +2 -0
- package/data/google-crawlers.json +7 -1
- package/getswagger.sh +40 -4
- package/gettext-extract.js +163 -0
- package/lib/account.js +135 -72
- package/lib/api-routes/account-routes.js +684 -106
- package/lib/api-routes/blocklist-routes.js +344 -0
- package/lib/api-routes/chat-routes.js +32 -14
- package/lib/api-routes/delivery-test-routes.js +346 -0
- package/lib/api-routes/export-routes.js +28 -14
- package/lib/api-routes/gateway-routes.js +427 -0
- package/lib/api-routes/license-routes.js +156 -0
- package/lib/api-routes/mailbox-routes.js +344 -0
- package/lib/api-routes/message-routes.js +221 -187
- package/lib/api-routes/oauth2-app-routes.js +697 -0
- package/lib/api-routes/outbox-routes.js +185 -0
- package/lib/api-routes/pubsub-routes.js +102 -0
- package/lib/api-routes/route-helpers.js +58 -0
- package/lib/api-routes/settings-routes.js +357 -0
- package/lib/api-routes/stats-routes.js +111 -0
- package/lib/api-routes/submit-routes.js +461 -0
- package/lib/api-routes/template-routes.js +60 -75
- package/lib/api-routes/token-routes.js +297 -0
- package/lib/api-routes/webhook-route-routes.js +181 -0
- package/lib/autodetect-imap-settings.js +0 -2
- package/lib/consts.js +5 -0
- package/lib/email-client/base-client.js +28 -6
- package/lib/email-client/gmail-client.js +133 -112
- package/lib/email-client/imap/mailbox.js +34 -11
- package/lib/email-client/imap/subconnection.js +20 -13
- package/lib/email-client/imap/sync-operations.js +131 -3
- package/lib/email-client/imap-client.js +152 -75
- package/lib/email-client/notification-handler.js +1 -4
- package/lib/email-client/outlook-client.js +134 -75
- package/lib/export.js +97 -20
- package/lib/feature-flags.js +2 -2
- package/lib/gateway.js +4 -9
- package/lib/get-raw-email.js +5 -5
- package/lib/imapproxy/imap-core/lib/commands/starttls.js +18 -0
- package/lib/imapproxy/imap-core/lib/imap-command.js +6 -1
- package/lib/imapproxy/imap-core/lib/imap-connection.js +106 -24
- package/lib/imapproxy/imap-core/lib/imap-server.js +24 -0
- package/lib/imapproxy/imap-core/lib/imap-stream.js +26 -0
- package/lib/logger.js +24 -21
- package/lib/message-port-stream.js +113 -16
- package/lib/metrics-collector.js +0 -2
- package/lib/oauth2-apps.js +13 -4
- package/lib/outbox.js +24 -40
- package/lib/redis-operations.js +1 -1
- package/lib/reject-worker-calls.js +42 -0
- package/lib/routes-ui.js +37 -8778
- package/lib/schemas.js +429 -84
- package/lib/sentry.js +139 -0
- package/lib/settings.js +9 -3
- package/lib/stream-encrypt.js +1 -1
- package/lib/templates.js +1 -1
- package/lib/tokens.js +5 -3
- package/lib/tools.js +70 -4
- package/lib/ui-routes/account-routes.js +45 -212
- package/lib/ui-routes/admin-config-routes.js +928 -489
- package/lib/ui-routes/admin-entities-routes.js +1 -0
- package/lib/ui-routes/auth-routes.js +1339 -0
- package/lib/ui-routes/dashboard-routes.js +188 -0
- package/lib/ui-routes/document-store-routes.js +800 -0
- package/lib/ui-routes/export-routes.js +217 -0
- package/lib/ui-routes/internals-routes.js +354 -0
- package/lib/ui-routes/network-config-routes.js +759 -0
- package/lib/ui-routes/{oauth-routes.js → oauth-config-routes.js} +369 -91
- package/lib/ui-routes/route-helpers.js +314 -0
- package/lib/ui-routes/smtp-test-routes.js +236 -0
- package/lib/ui-routes/unsubscribe-routes.js +232 -0
- package/lib/webhook-request.js +36 -0
- package/lib/webhooks.js +8 -4
- package/package.json +13 -12
- package/sbom.json +1 -1
- package/server.js +222 -39
- package/static/licenses.html +160 -300
- package/translations/messages.pot +112 -132
- package/update-info.sh +19 -1
- package/views/config/logging.hbs +48 -0
- package/views/dashboard.hbs +7 -26
- package/views/internals/index.hbs +15 -0
- package/views/tokens/index.hbs +9 -0
- package/workers/api.js +200 -4424
- package/workers/documents.js +2 -22
- package/workers/export.js +103 -104
- package/workers/imap-proxy.js +3 -23
- package/workers/imap.js +32 -36
- package/workers/smtp.js +2 -22
- package/workers/submit.js +26 -35
- package/workers/webhooks.js +9 -43
package/workers/documents.js
CHANGED
|
@@ -14,28 +14,8 @@ const GB_COLLECT_DELAY = 6 * 3600 * 1000; // 6h
|
|
|
14
14
|
const GB_FAILURE_DELAY = 3 * 1000;
|
|
15
15
|
const GB_EMPTY_DELAY = 10 * 1000;
|
|
16
16
|
|
|
17
|
-
const
|
|
18
|
-
|
|
19
|
-
Bugsnag.start({
|
|
20
|
-
apiKey: readEnvValue('BUGSNAG_API_KEY'),
|
|
21
|
-
appVersion: packageData.version,
|
|
22
|
-
logger: {
|
|
23
|
-
debug(...args) {
|
|
24
|
-
logger.debug({ msg: args.shift(), worker: 'documents', source: 'bugsnag', args: args.length ? args : undefined });
|
|
25
|
-
},
|
|
26
|
-
info(...args) {
|
|
27
|
-
logger.debug({ msg: args.shift(), worker: 'documents', source: 'bugsnag', args: args.length ? args : undefined });
|
|
28
|
-
},
|
|
29
|
-
warn(...args) {
|
|
30
|
-
logger.warn({ msg: args.shift(), worker: 'documents', source: 'bugsnag', args: args.length ? args : undefined });
|
|
31
|
-
},
|
|
32
|
-
error(...args) {
|
|
33
|
-
logger.error({ msg: args.shift(), worker: 'documents', source: 'bugsnag', args: args.length ? args : undefined });
|
|
34
|
-
}
|
|
35
|
-
}
|
|
36
|
-
});
|
|
37
|
-
logger.notifyError = Bugsnag.notify.bind(Bugsnag);
|
|
38
|
-
}
|
|
17
|
+
const { initSentry } = require('../lib/sentry');
|
|
18
|
+
initSentry('documents');
|
|
39
19
|
|
|
40
20
|
const { redis, queueConf } = require('../lib/db');
|
|
41
21
|
const { Worker } = require('bullmq');
|
package/workers/export.js
CHANGED
|
@@ -17,33 +17,13 @@ const {
|
|
|
17
17
|
DEFAULT_EXPORT_MAX_MESSAGES,
|
|
18
18
|
DEFAULT_EXPORT_MAX_SIZE
|
|
19
19
|
} = require('../lib/consts');
|
|
20
|
-
const { getDuration, readEnvValue, threadStats,
|
|
20
|
+
const { getDuration, readEnvValue, threadStats, maybeReloadHttpProxyAgent } = require('../lib/tools');
|
|
21
21
|
const { webhooks: Webhooks } = require('../lib/webhooks');
|
|
22
22
|
const settings = require('../lib/settings');
|
|
23
|
-
const { Export } = require('../lib/export');
|
|
24
|
-
|
|
25
|
-
const
|
|
26
|
-
|
|
27
|
-
Bugsnag.start({
|
|
28
|
-
apiKey: readEnvValue('BUGSNAG_API_KEY'),
|
|
29
|
-
appVersion: packageData.version,
|
|
30
|
-
logger: {
|
|
31
|
-
debug(...args) {
|
|
32
|
-
logger.debug({ msg: args.shift(), worker: 'export', source: 'bugsnag', args: args.length ? args : undefined });
|
|
33
|
-
},
|
|
34
|
-
info(...args) {
|
|
35
|
-
logger.debug({ msg: args.shift(), worker: 'export', source: 'bugsnag', args: args.length ? args : undefined });
|
|
36
|
-
},
|
|
37
|
-
warn(...args) {
|
|
38
|
-
logger.warn({ msg: args.shift(), worker: 'export', source: 'bugsnag', args: args.length ? args : undefined });
|
|
39
|
-
},
|
|
40
|
-
error(...args) {
|
|
41
|
-
logger.error({ msg: args.shift(), worker: 'export', source: 'bugsnag', args: args.length ? args : undefined });
|
|
42
|
-
}
|
|
43
|
-
}
|
|
44
|
-
});
|
|
45
|
-
logger.notifyError = Bugsnag.notify.bind(Bugsnag);
|
|
46
|
-
}
|
|
23
|
+
const { Export, isTransientError, isSkippableError, isFolderMissingError } = require('../lib/export');
|
|
24
|
+
|
|
25
|
+
const { initSentry } = require('../lib/sentry');
|
|
26
|
+
initSentry('export');
|
|
47
27
|
|
|
48
28
|
const { redis, queueConf } = require('../lib/db');
|
|
49
29
|
const { Worker } = require('bullmq');
|
|
@@ -74,23 +54,6 @@ const IMAP_MESSAGE_RETRY_BASE_DELAY = 2000;
|
|
|
74
54
|
const ACCOUNT_CHECK_INTERVAL = 60 * 1000;
|
|
75
55
|
const LOCK_EXTENSION_INTERVAL = 5 * 60 * 1000;
|
|
76
56
|
|
|
77
|
-
function isTransientError(err) {
|
|
78
|
-
if (['ETIMEDOUT', 'ECONNRESET', 'ENOTFOUND', 'EAI_AGAIN', 'ECONNREFUSED', 'EPIPE', 'EHOSTUNREACH'].includes(err.code)) {
|
|
79
|
-
return true;
|
|
80
|
-
}
|
|
81
|
-
if (err.statusCode >= 500 && err.statusCode < 600) {
|
|
82
|
-
return true;
|
|
83
|
-
}
|
|
84
|
-
if (err.code === 'Timeout' || err.message?.includes('timeout')) {
|
|
85
|
-
return true;
|
|
86
|
-
}
|
|
87
|
-
return false;
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
function isSkippableError(err) {
|
|
91
|
-
return err.code === 'MessageNotFound' || err.statusCode === 404 || err.message?.includes('Failed to generate message ID');
|
|
92
|
-
}
|
|
93
|
-
|
|
94
57
|
let callQueue = new Map();
|
|
95
58
|
let mids = 0;
|
|
96
59
|
|
|
@@ -155,6 +118,28 @@ async function notify(account, event, data) {
|
|
|
155
118
|
await Webhooks.pushToQueue(event, await Webhooks.formatPayload(event, payload));
|
|
156
119
|
}
|
|
157
120
|
|
|
121
|
+
// Returns a throttled callback that extends both the BullMQ job lock and the export's Redis key
|
|
122
|
+
// expiry on the LOCK_EXTENSION_INTERVAL cadence. Long-running indexing/exporting loops call it
|
|
123
|
+
// freely; the lock/expiry are refreshed at most once per interval. Without this the lock could lapse
|
|
124
|
+
// (stalling and reprocessing the job) and the export keys could expire mid-run.
|
|
125
|
+
function createLeaseExtender(job, account, exportId) {
|
|
126
|
+
let lastExtension = Date.now();
|
|
127
|
+
return async function maybeExtendLease() {
|
|
128
|
+
if (Date.now() - lastExtension <= LOCK_EXTENSION_INTERVAL) {
|
|
129
|
+
return;
|
|
130
|
+
}
|
|
131
|
+
await Promise.all([
|
|
132
|
+
job.extendLock(job.token, 10 * 60 * 1000).catch(err => {
|
|
133
|
+
logger.warn({ msg: 'Failed to extend job lock', account, exportId, err });
|
|
134
|
+
}),
|
|
135
|
+
Export.extendExpiry(account, exportId).catch(err => {
|
|
136
|
+
logger.warn({ msg: 'Failed to extend export expiry', account, exportId, err });
|
|
137
|
+
})
|
|
138
|
+
]);
|
|
139
|
+
lastExtension = Date.now();
|
|
140
|
+
};
|
|
141
|
+
}
|
|
142
|
+
|
|
158
143
|
async function indexMessages(job, exportData) {
|
|
159
144
|
const { account, exportId } = job.data;
|
|
160
145
|
const folders = JSON.parse(exportData.folders || '[]');
|
|
@@ -195,15 +180,10 @@ async function indexMessages(job, exportData) {
|
|
|
195
180
|
|
|
196
181
|
let totalIndexed = 0;
|
|
197
182
|
let truncated = false;
|
|
198
|
-
|
|
183
|
+
const maybeExtendLease = createLeaseExtender(job, account, exportId);
|
|
199
184
|
|
|
200
185
|
for (let i = 0; i < foldersToProcess.length; i++) {
|
|
201
|
-
|
|
202
|
-
await job.extendLock(job.token, 10 * 60 * 1000).catch(err => {
|
|
203
|
-
logger.warn({ msg: 'Failed to extend job lock during indexing', account, exportId, err });
|
|
204
|
-
});
|
|
205
|
-
lastLockExtension = Date.now();
|
|
206
|
-
}
|
|
186
|
+
await maybeExtendLease();
|
|
207
187
|
|
|
208
188
|
if (await Export.isCancelled(account, exportId)) {
|
|
209
189
|
const err = new Error('Export cancelled by user');
|
|
@@ -221,8 +201,6 @@ async function indexMessages(job, exportData) {
|
|
|
221
201
|
const queued = await indexFolder(accountObject, account, exportId, folderPath, startDate, endDate, indexingStartTime, remaining);
|
|
222
202
|
totalIndexed += queued;
|
|
223
203
|
|
|
224
|
-
await Export.update(account, exportId, { foldersScanned: i + 1 });
|
|
225
|
-
|
|
226
204
|
logger.trace({
|
|
227
205
|
msg: 'Folder indexed',
|
|
228
206
|
account,
|
|
@@ -266,6 +244,9 @@ async function indexMessages(job, exportData) {
|
|
|
266
244
|
});
|
|
267
245
|
}
|
|
268
246
|
|
|
247
|
+
// Count the folder as scanned whether or not it succeeded so progress reaches foldersTotal.
|
|
248
|
+
await Export.update(account, exportId, { foldersScanned: i + 1 });
|
|
249
|
+
|
|
269
250
|
if (maxMessages && totalIndexed >= maxMessages) {
|
|
270
251
|
truncated = true;
|
|
271
252
|
logger.warn({
|
|
@@ -323,7 +304,18 @@ async function indexFolder(accountObject, account, exportId, folderPath, startDa
|
|
|
323
304
|
cursor
|
|
324
305
|
};
|
|
325
306
|
|
|
326
|
-
|
|
307
|
+
let result;
|
|
308
|
+
try {
|
|
309
|
+
result = await accountObject.listMessages(listOptions);
|
|
310
|
+
} catch (err) {
|
|
311
|
+
// Other errors (e.g. a deleted account) propagate
|
|
312
|
+
if (isFolderMissingError(err)) {
|
|
313
|
+
// The folder disappeared mid-export - treat it as empty instead of failing the export
|
|
314
|
+
logger.warn({ msg: 'Export folder was not found, skipping', account, exportId, folder: folderPath, err });
|
|
315
|
+
return queued;
|
|
316
|
+
}
|
|
317
|
+
throw err;
|
|
318
|
+
}
|
|
327
319
|
|
|
328
320
|
for (const msg of result.messages || []) {
|
|
329
321
|
if (maxMessages && queued >= maxMessages) {
|
|
@@ -357,7 +349,9 @@ async function exportMessages(job, exportData) {
|
|
|
357
349
|
const { filePath } = exportData;
|
|
358
350
|
const includeAttachments = exportData.includeAttachments === '1';
|
|
359
351
|
const textType = exportData.textType || '*';
|
|
360
|
-
const
|
|
352
|
+
const rawMaxBytes = Number(exportData.maxBytes);
|
|
353
|
+
// Preserve an explicit 0 ("unlimited" per the API contract); only fall back when unset/invalid.
|
|
354
|
+
const maxBytes = Number.isFinite(rawMaxBytes) ? rawMaxBytes : 5 * 1024 * 1024;
|
|
361
355
|
const maxMessageSize = (await settings.get('exportMaxMessageSize')) || DEFAULT_EXPORT_MAX_MESSAGE_SIZE;
|
|
362
356
|
const maxExportSize = Number(await settings.get('exportMaxSize')) || DEFAULT_EXPORT_MAX_SIZE;
|
|
363
357
|
const isEncrypted = exportData.isEncrypted === '1';
|
|
@@ -419,14 +413,15 @@ async function exportMessages(job, exportData) {
|
|
|
419
413
|
});
|
|
420
414
|
}
|
|
421
415
|
|
|
422
|
-
let lastScore = Number(exportData.lastProcessedScore) || 0;
|
|
423
416
|
let processed = 0;
|
|
424
417
|
let totalBytesWritten = 0;
|
|
425
418
|
let processingError = null;
|
|
426
419
|
let sizeLimitReached = false;
|
|
427
420
|
|
|
428
421
|
let lastAccountCheck = Date.now();
|
|
429
|
-
|
|
422
|
+
// Refreshes the job lock and export key expiry on a fixed cadence; called from the main loop and
|
|
423
|
+
// from inside the rate-limit backoff so a batch stuck retrying does not let the lock lapse.
|
|
424
|
+
const maybeExtendLease = createLeaseExtender(job, account, exportId);
|
|
430
425
|
|
|
431
426
|
const accountData = await accountObject.loadAccountData(account);
|
|
432
427
|
const isApiAccount = await accountObject.isApiClient(accountData);
|
|
@@ -490,12 +485,7 @@ async function exportMessages(job, exportData) {
|
|
|
490
485
|
break;
|
|
491
486
|
}
|
|
492
487
|
|
|
493
|
-
|
|
494
|
-
await job.extendLock(job.token, 10 * 60 * 1000).catch(err => {
|
|
495
|
-
logger.warn({ msg: 'Failed to extend job lock', account, exportId, err });
|
|
496
|
-
});
|
|
497
|
-
lastLockExtension = Date.now();
|
|
498
|
-
}
|
|
488
|
+
await maybeExtendLease();
|
|
499
489
|
|
|
500
490
|
if (await Export.isCancelled(account, exportId)) {
|
|
501
491
|
const err = new Error('Export cancelled by user');
|
|
@@ -513,7 +503,7 @@ async function exportMessages(job, exportData) {
|
|
|
513
503
|
lastAccountCheck = Date.now();
|
|
514
504
|
}
|
|
515
505
|
|
|
516
|
-
const batch = await Export.getNextBatch(account, exportId,
|
|
506
|
+
const batch = await Export.getNextBatch(account, exportId, BATCH_SIZE);
|
|
517
507
|
if (batch.length === 0) {
|
|
518
508
|
break;
|
|
519
509
|
}
|
|
@@ -522,14 +512,12 @@ async function exportMessages(job, exportData) {
|
|
|
522
512
|
for (const entry of batch) {
|
|
523
513
|
if (includeAttachments && entry.size > maxMessageSize) {
|
|
524
514
|
await Export.incrementSkipped(account, exportId);
|
|
525
|
-
lastScore = entry.score;
|
|
526
515
|
} else {
|
|
527
516
|
entriesToFetch.push(entry);
|
|
528
517
|
}
|
|
529
518
|
}
|
|
530
519
|
|
|
531
520
|
if (entriesToFetch.length === 0) {
|
|
532
|
-
await Export.updateLastProcessedScore(account, exportId, lastScore);
|
|
533
521
|
continue;
|
|
534
522
|
}
|
|
535
523
|
|
|
@@ -573,7 +561,6 @@ async function exportMessages(job, exportData) {
|
|
|
573
561
|
reason: err.message || err.code
|
|
574
562
|
});
|
|
575
563
|
await Export.incrementSkipped(account, exportId);
|
|
576
|
-
lastScore = entry.score;
|
|
577
564
|
} else if (isRateLimited && rateLimitRetry < MAX_RATE_LIMIT_RETRIES) {
|
|
578
565
|
rateLimitedEntries.push(entry);
|
|
579
566
|
} else {
|
|
@@ -584,7 +571,6 @@ async function exportMessages(job, exportData) {
|
|
|
584
571
|
}
|
|
585
572
|
} else if (result && result.data) {
|
|
586
573
|
await processMessage(result.data, entry);
|
|
587
|
-
lastScore = entry.score;
|
|
588
574
|
if (maxExportSize && totalBytesWritten >= maxExportSize) {
|
|
589
575
|
sizeLimitReached = true;
|
|
590
576
|
break;
|
|
@@ -599,7 +585,6 @@ async function exportMessages(job, exportData) {
|
|
|
599
585
|
reason: 'Message not found in batch results'
|
|
600
586
|
});
|
|
601
587
|
await Export.incrementSkipped(account, exportId);
|
|
602
|
-
lastScore = entry.score;
|
|
603
588
|
}
|
|
604
589
|
}
|
|
605
590
|
|
|
@@ -620,6 +605,7 @@ async function exportMessages(job, exportData) {
|
|
|
620
605
|
delayMs: Math.round(delay)
|
|
621
606
|
});
|
|
622
607
|
await new Promise(resolve => setTimeout(resolve, delay));
|
|
608
|
+
await maybeExtendLease();
|
|
623
609
|
fetchBatch = rateLimitedEntries;
|
|
624
610
|
} else {
|
|
625
611
|
break;
|
|
@@ -672,7 +658,6 @@ async function exportMessages(job, exportData) {
|
|
|
672
658
|
await processMessage(message, entry);
|
|
673
659
|
if (maxExportSize && totalBytesWritten >= maxExportSize) {
|
|
674
660
|
sizeLimitReached = true;
|
|
675
|
-
lastScore = entry.score;
|
|
676
661
|
break;
|
|
677
662
|
}
|
|
678
663
|
} else if (fetchError && isSkippableError(fetchError)) {
|
|
@@ -688,12 +673,9 @@ async function exportMessages(job, exportData) {
|
|
|
688
673
|
} else if (fetchError) {
|
|
689
674
|
throw fetchError;
|
|
690
675
|
}
|
|
691
|
-
|
|
692
|
-
lastScore = entry.score;
|
|
693
676
|
}
|
|
694
677
|
}
|
|
695
678
|
|
|
696
|
-
await Export.updateLastProcessedScore(account, exportId, lastScore);
|
|
697
679
|
logger.trace({ msg: 'Export batch processed', account, exportId, messagesExported: processed });
|
|
698
680
|
}
|
|
699
681
|
} catch (err) {
|
|
@@ -764,19 +746,26 @@ const exportWorker = new Worker(
|
|
|
764
746
|
|
|
765
747
|
await Export.complete(account, exportId);
|
|
766
748
|
|
|
767
|
-
|
|
749
|
+
// The export is complete and persisted. A failure while reading final stats or delivering
|
|
750
|
+
// the completion webhook must NOT fall through to the job catch below, which would delete
|
|
751
|
+
// the finished file and mark the export as failed.
|
|
752
|
+
try {
|
|
753
|
+
const finalData = await redis.hgetall(`${REDIS_PREFIX}exp:${account}:${exportId}`);
|
|
768
754
|
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
755
|
+
await notify(account, EXPORT_COMPLETED_NOTIFY, {
|
|
756
|
+
exportId,
|
|
757
|
+
folders: JSON.parse(exportData.folders || '[]'),
|
|
758
|
+
startDate: new Date(Number(exportData.startDate)).toISOString(),
|
|
759
|
+
endDate: new Date(Number(exportData.endDate)).toISOString(),
|
|
760
|
+
messagesExported: Number(finalData.messagesExported) || 0,
|
|
761
|
+
messagesSkipped: Number(finalData.messagesSkipped) || 0,
|
|
762
|
+
bytesWritten: Number(finalData.bytesWritten) || 0,
|
|
763
|
+
duration: Date.now() - startTime,
|
|
764
|
+
expiresAt: new Date(Number(finalData.expiresAt)).toISOString()
|
|
765
|
+
});
|
|
766
|
+
} catch (notifyErr) {
|
|
767
|
+
logger.error({ msg: 'Failed to deliver export completion notification', account, exportId, err: notifyErr });
|
|
768
|
+
}
|
|
780
769
|
|
|
781
770
|
logger.info({ msg: 'Export job completed', account, exportId, duration: Date.now() - startTime });
|
|
782
771
|
} catch (err) {
|
|
@@ -795,14 +784,19 @@ const exportWorker = new Worker(
|
|
|
795
784
|
}
|
|
796
785
|
|
|
797
786
|
if (err.code !== 'AccountDeleted' && err.code !== 'AccountNotFound' && err.code !== 'ExportCancelled') {
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
787
|
+
try {
|
|
788
|
+
await notify(account, EXPORT_FAILED_NOTIFY, {
|
|
789
|
+
exportId,
|
|
790
|
+
error: err.message,
|
|
791
|
+
errorCode: err.code,
|
|
792
|
+
phase: exportData.phase || 'unknown',
|
|
793
|
+
messagesExported: Number(exportData.messagesExported) || 0,
|
|
794
|
+
messagesQueued: Number(exportData.messagesQueued) || 0
|
|
795
|
+
});
|
|
796
|
+
} catch (notifyErr) {
|
|
797
|
+
// Never let a failed notification replace the original error below.
|
|
798
|
+
logger.error({ msg: 'Failed to deliver export failure notification', account, exportId, err: notifyErr });
|
|
799
|
+
}
|
|
806
800
|
}
|
|
807
801
|
|
|
808
802
|
throw err;
|
|
@@ -876,7 +870,7 @@ function onCommand(command) {
|
|
|
876
870
|
try {
|
|
877
871
|
const activeExports = await redis.smembers(`${REDIS_PREFIX}exp:active`);
|
|
878
872
|
for (const entry of activeExports) {
|
|
879
|
-
const separatorIndex = entry.
|
|
873
|
+
const separatorIndex = entry.lastIndexOf(':exp_');
|
|
880
874
|
if (separatorIndex === -1) continue;
|
|
881
875
|
const entryAccount = entry.substring(0, separatorIndex);
|
|
882
876
|
const entryExportId = entry.substring(separatorIndex + 1);
|
|
@@ -893,6 +887,22 @@ function onCommand(command) {
|
|
|
893
887
|
5 * 60 * 1000
|
|
894
888
|
).unref();
|
|
895
889
|
|
|
890
|
+
// Periodically remove export files whose Redis state has already expired. Without this, files of
|
|
891
|
+
// expired exports would only be reclaimed on the next worker restart.
|
|
892
|
+
setInterval(
|
|
893
|
+
async () => {
|
|
894
|
+
try {
|
|
895
|
+
const cleaned = await Export.cleanup();
|
|
896
|
+
if (cleaned > 0) {
|
|
897
|
+
logger.info({ msg: 'Cleaned up orphaned export files', count: cleaned });
|
|
898
|
+
}
|
|
899
|
+
} catch (err) {
|
|
900
|
+
logger.error({ msg: 'Failed to clean up export files', err });
|
|
901
|
+
}
|
|
902
|
+
},
|
|
903
|
+
60 * 60 * 1000
|
|
904
|
+
).unref();
|
|
905
|
+
|
|
896
906
|
parentPort.postMessage({ cmd: 'ready' });
|
|
897
907
|
})();
|
|
898
908
|
|
|
@@ -919,19 +929,8 @@ parentPort.on('message', message => {
|
|
|
919
929
|
}
|
|
920
930
|
|
|
921
931
|
if (message && message.cmd === 'settings') {
|
|
922
|
-
|
|
923
|
-
if ('httpProxyEnabled' in d || 'httpProxyUrl' in d) {
|
|
924
|
-
reloadHttpProxyAgent().catch(err => logger.error({ msg: 'Failed to reload HTTP proxy agent', err }));
|
|
925
|
-
}
|
|
932
|
+
maybeReloadHttpProxyAgent(message.data);
|
|
926
933
|
}
|
|
927
934
|
});
|
|
928
935
|
|
|
929
936
|
logger.info({ msg: 'Started export worker thread', version: packageData.version });
|
|
930
|
-
|
|
931
|
-
module.exports = {
|
|
932
|
-
isTransientError,
|
|
933
|
-
isSkippableError,
|
|
934
|
-
IMAP_MESSAGE_MAX_RETRIES,
|
|
935
|
-
IMAP_MESSAGE_RETRY_BASE_DELAY,
|
|
936
|
-
ACCOUNT_CHECK_INTERVAL
|
|
937
|
-
};
|
package/workers/imap-proxy.js
CHANGED
|
@@ -5,32 +5,12 @@ const { parentPort } = require('worker_threads');
|
|
|
5
5
|
const packageData = require('../package.json');
|
|
6
6
|
const logger = require('../lib/logger');
|
|
7
7
|
|
|
8
|
-
const {
|
|
8
|
+
const { threadStats } = require('../lib/tools');
|
|
9
9
|
|
|
10
10
|
const { run } = require('../lib/imapproxy/imap-server');
|
|
11
11
|
|
|
12
|
-
const
|
|
13
|
-
|
|
14
|
-
Bugsnag.start({
|
|
15
|
-
apiKey: readEnvValue('BUGSNAG_API_KEY'),
|
|
16
|
-
appVersion: packageData.version,
|
|
17
|
-
logger: {
|
|
18
|
-
debug(...args) {
|
|
19
|
-
logger.debug({ msg: args.shift(), worker: 'imapProxy', source: 'bugsnag', args: args.length ? args : undefined });
|
|
20
|
-
},
|
|
21
|
-
info(...args) {
|
|
22
|
-
logger.debug({ msg: args.shift(), worker: 'imapProxy', source: 'bugsnag', args: args.length ? args : undefined });
|
|
23
|
-
},
|
|
24
|
-
warn(...args) {
|
|
25
|
-
logger.warn({ msg: args.shift(), worker: 'imapProxy', source: 'bugsnag', args: args.length ? args : undefined });
|
|
26
|
-
},
|
|
27
|
-
error(...args) {
|
|
28
|
-
logger.error({ msg: args.shift(), worker: 'imapProxy', source: 'bugsnag', args: args.length ? args : undefined });
|
|
29
|
-
}
|
|
30
|
-
}
|
|
31
|
-
});
|
|
32
|
-
logger.notifyError = Bugsnag.notify.bind(Bugsnag);
|
|
33
|
-
}
|
|
12
|
+
const { initSentry } = require('../lib/sentry');
|
|
13
|
+
initSentry('imapProxy');
|
|
34
14
|
|
|
35
15
|
async function onCommand(command) {
|
|
36
16
|
switch (command.cmd) {
|
package/workers/imap.js
CHANGED
|
@@ -7,30 +7,10 @@ const logger = require('../lib/logger');
|
|
|
7
7
|
|
|
8
8
|
const { REDIS_PREFIX } = require('../lib/consts');
|
|
9
9
|
|
|
10
|
-
const { getDuration, getBoolean, emitChangeEvent, readEnvValue, hasEnvValue, threadStats,
|
|
11
|
-
|
|
12
|
-
const
|
|
13
|
-
|
|
14
|
-
Bugsnag.start({
|
|
15
|
-
apiKey: readEnvValue('BUGSNAG_API_KEY'),
|
|
16
|
-
appVersion: packageData.version,
|
|
17
|
-
logger: {
|
|
18
|
-
debug(...args) {
|
|
19
|
-
logger.debug({ msg: args.shift(), worker: 'imap', source: 'bugsnag', args: args.length ? args : undefined });
|
|
20
|
-
},
|
|
21
|
-
info(...args) {
|
|
22
|
-
logger.debug({ msg: args.shift(), worker: 'imap', source: 'bugsnag', args: args.length ? args : undefined });
|
|
23
|
-
},
|
|
24
|
-
warn(...args) {
|
|
25
|
-
logger.warn({ msg: args.shift(), worker: 'imap', source: 'bugsnag', args: args.length ? args : undefined });
|
|
26
|
-
},
|
|
27
|
-
error(...args) {
|
|
28
|
-
logger.error({ msg: args.shift(), worker: 'imap', source: 'bugsnag', args: args.length ? args : undefined });
|
|
29
|
-
}
|
|
30
|
-
}
|
|
31
|
-
});
|
|
32
|
-
logger.notifyError = Bugsnag.notify.bind(Bugsnag);
|
|
33
|
-
}
|
|
10
|
+
const { getDuration, getBoolean, emitChangeEvent, readEnvValue, hasEnvValue, threadStats, maybeReloadHttpProxyAgent } = require('../lib/tools');
|
|
11
|
+
|
|
12
|
+
const { initSentry } = require('../lib/sentry');
|
|
13
|
+
initSentry('imap');
|
|
34
14
|
|
|
35
15
|
const { IMAPClient } = require('../lib/email-client/imap-client');
|
|
36
16
|
const { GmailClient } = require('../lib/email-client/gmail-client');
|
|
@@ -39,7 +19,7 @@ const { BaseClient } = require('../lib/email-client/base-client');
|
|
|
39
19
|
const { Account } = require('../lib/account');
|
|
40
20
|
const { oauth2Apps, isApiBasedApp } = require('../lib/oauth2-apps');
|
|
41
21
|
const { redis, notifyQueue, submitQueue, documentsQueue, getFlowProducer } = require('../lib/db');
|
|
42
|
-
const { MessagePortWritable } = require('../lib/message-port-stream');
|
|
22
|
+
const { MessagePortWritable, pipeToMessagePort } = require('../lib/message-port-stream');
|
|
43
23
|
const { getESClient } = require('../lib/document-store');
|
|
44
24
|
const settings = require('../lib/settings');
|
|
45
25
|
const msgpack = require('msgpack5')();
|
|
@@ -728,8 +708,6 @@ class ConnectionHandler {
|
|
|
728
708
|
if (!accountData.connection) {
|
|
729
709
|
throw NO_ACTIVE_HANDLER_RESP_ERR;
|
|
730
710
|
}
|
|
731
|
-
let stream = new MessagePortWritable(message.port);
|
|
732
|
-
|
|
733
711
|
let source = await accountData.connection.getRawMessage(message.message);
|
|
734
712
|
if (!source) {
|
|
735
713
|
let err = new Error('Requested file not found');
|
|
@@ -737,11 +715,22 @@ class ConnectionHandler {
|
|
|
737
715
|
throw err;
|
|
738
716
|
}
|
|
739
717
|
|
|
718
|
+
// Build the cross-thread writable only after we know there is content to send.
|
|
719
|
+
// Constructing it earlier leaks the transferred MessagePort (and its message
|
|
720
|
+
// listener) whenever the lookup above throws (e.g. a 404).
|
|
721
|
+
let stream = new MessagePortWritable(message.port);
|
|
722
|
+
|
|
740
723
|
setImmediate(() => {
|
|
741
724
|
if (Buffer.isBuffer(source)) {
|
|
742
|
-
|
|
725
|
+
// The consumer may have aborted and destroyed the writable (via a cancel
|
|
726
|
+
// message) before this runs; ending a destroyed stream would emit an
|
|
727
|
+
// unhandled 'error' and take down the worker. The streaming branch is
|
|
728
|
+
// safe because pipeline() tears down a destroyed destination cleanly.
|
|
729
|
+
if (!stream.destroyed) {
|
|
730
|
+
stream.end(source);
|
|
731
|
+
}
|
|
743
732
|
} else {
|
|
744
|
-
source.
|
|
733
|
+
pipeToMessagePort(source, stream, accountData.connection.logger);
|
|
745
734
|
}
|
|
746
735
|
});
|
|
747
736
|
|
|
@@ -764,8 +753,6 @@ class ConnectionHandler {
|
|
|
764
753
|
throw NO_ACTIVE_HANDLER_RESP_ERR;
|
|
765
754
|
}
|
|
766
755
|
|
|
767
|
-
let stream = new MessagePortWritable(message.port);
|
|
768
|
-
|
|
769
756
|
let source = await accountData.connection.getAttachment(message.attachment);
|
|
770
757
|
if (!source) {
|
|
771
758
|
let err = new Error('Requested file not found');
|
|
@@ -773,11 +760,22 @@ class ConnectionHandler {
|
|
|
773
760
|
throw err;
|
|
774
761
|
}
|
|
775
762
|
|
|
763
|
+
// Build the cross-thread writable only after we know there is content to send.
|
|
764
|
+
// Constructing it earlier leaks the transferred MessagePort (and its message
|
|
765
|
+
// listener) whenever the lookup above throws (e.g. a 404).
|
|
766
|
+
let stream = new MessagePortWritable(message.port);
|
|
767
|
+
|
|
776
768
|
setImmediate(() => {
|
|
777
769
|
if (Buffer.isBuffer(source.data)) {
|
|
778
|
-
|
|
770
|
+
// The consumer may have aborted and destroyed the writable (via a cancel
|
|
771
|
+
// message) before this runs; ending a destroyed stream would emit an
|
|
772
|
+
// unhandled 'error' and take down the worker. The streaming branch is
|
|
773
|
+
// safe because pipeline() tears down a destroyed destination cleanly.
|
|
774
|
+
if (!stream.destroyed) {
|
|
775
|
+
stream.end(source.data);
|
|
776
|
+
}
|
|
779
777
|
} else {
|
|
780
|
-
source.
|
|
778
|
+
pipeToMessagePort(source, stream, accountData.connection.logger);
|
|
781
779
|
}
|
|
782
780
|
});
|
|
783
781
|
|
|
@@ -812,9 +810,7 @@ class ConnectionHandler {
|
|
|
812
810
|
|
|
813
811
|
switch (message.cmd) {
|
|
814
812
|
case 'settings':
|
|
815
|
-
|
|
816
|
-
reloadHttpProxyAgent().catch(err => logger.error({ msg: 'Failed to reload HTTP proxy agent', err }));
|
|
817
|
-
}
|
|
813
|
+
maybeReloadHttpProxyAgent(message.data);
|
|
818
814
|
|
|
819
815
|
if (message.data && message.data.logs) {
|
|
820
816
|
for (let [account, accountObject] of this.accounts) {
|
package/workers/smtp.js
CHANGED
|
@@ -9,28 +9,8 @@ const logger = require('../lib/logger');
|
|
|
9
9
|
const { getDuration, emitChangeEvent, readEnvValue, threadStats, loadTlsConfig, getByteSize } = require('../lib/tools');
|
|
10
10
|
const { matchIp } = require('../lib/utils/network');
|
|
11
11
|
|
|
12
|
-
const
|
|
13
|
-
|
|
14
|
-
Bugsnag.start({
|
|
15
|
-
apiKey: readEnvValue('BUGSNAG_API_KEY'),
|
|
16
|
-
appVersion: packageData.version,
|
|
17
|
-
logger: {
|
|
18
|
-
debug(...args) {
|
|
19
|
-
logger.debug({ msg: args.shift(), worker: 'smtp', source: 'bugsnag', args: args.length ? args : undefined });
|
|
20
|
-
},
|
|
21
|
-
info(...args) {
|
|
22
|
-
logger.debug({ msg: args.shift(), worker: 'smtp', source: 'bugsnag', args: args.length ? args : undefined });
|
|
23
|
-
},
|
|
24
|
-
warn(...args) {
|
|
25
|
-
logger.warn({ msg: args.shift(), worker: 'smtp', source: 'bugsnag', args: args.length ? args : undefined });
|
|
26
|
-
},
|
|
27
|
-
error(...args) {
|
|
28
|
-
logger.error({ msg: args.shift(), worker: 'smtp', source: 'bugsnag', args: args.length ? args : undefined });
|
|
29
|
-
}
|
|
30
|
-
}
|
|
31
|
-
});
|
|
32
|
-
logger.notifyError = Bugsnag.notify.bind(Bugsnag);
|
|
33
|
-
}
|
|
12
|
+
const { initSentry } = require('../lib/sentry');
|
|
13
|
+
initSentry('smtp');
|
|
34
14
|
|
|
35
15
|
const { SMTPServer } = require('smtp-server');
|
|
36
16
|
const util = require('util');
|