emailengine-app 2.61.4 → 2.62.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +87 -0
- package/data/google-crawlers.json +1 -1
- package/lib/account.js +20 -7
- package/lib/api-routes/account-routes.js +28 -5
- package/lib/api-routes/chat-routes.js +1 -1
- package/lib/api-routes/export-routes.js +316 -0
- package/lib/api-routes/message-routes.js +28 -23
- package/lib/api-routes/template-routes.js +28 -7
- package/lib/arf-detect.js +1 -1
- package/lib/consts.js +16 -0
- package/lib/db.js +3 -0
- package/lib/email-client/base-client.js +6 -4
- package/lib/email-client/gmail-client.js +204 -33
- package/lib/email-client/imap/mailbox.js +99 -8
- package/lib/email-client/imap/subconnection.js +5 -5
- package/lib/email-client/imap-client.js +76 -16
- package/lib/email-client/message-builder.js +3 -1
- package/lib/email-client/notification-handler.js +12 -9
- package/lib/email-client/outlook-client.js +362 -69
- package/lib/email-client/smtp-pool-manager.js +1 -1
- package/lib/export.js +528 -0
- package/lib/oauth/gmail.js +21 -13
- package/lib/oauth/mail-ru.js +23 -10
- package/lib/oauth/outlook.js +26 -16
- package/lib/oauth/pubsub/google.js +5 -0
- package/lib/routes-ui.js +236 -2
- package/lib/schemas.js +260 -80
- package/lib/stream-encrypt.js +263 -0
- package/lib/tools.js +30 -4
- package/lib/ui-routes/account-routes.js +24 -1
- package/lib/ui-routes/admin-config-routes.js +11 -4
- package/lib/ui-routes/admin-entities-routes.js +18 -0
- package/lib/webhooks.js +16 -20
- package/package.json +17 -17
- package/sbom.json +1 -1
- package/server.js +41 -5
- package/static/js/ace/ace.js +1 -1
- package/static/js/ace/ext-language_tools.js +1 -1
- package/static/licenses.html +47 -127
- package/translations/de.mo +0 -0
- package/translations/de.po +63 -36
- package/translations/en.mo +0 -0
- package/translations/en.po +64 -37
- package/translations/et.mo +0 -0
- package/translations/et.po +63 -36
- package/translations/fr.mo +0 -0
- package/translations/fr.po +63 -36
- package/translations/ja.mo +0 -0
- package/translations/ja.po +63 -36
- package/translations/messages.pot +88 -55
- package/translations/nl.mo +0 -0
- package/translations/nl.po +63 -36
- package/translations/pl.mo +0 -0
- package/translations/pl.po +63 -36
- package/views/accounts/account.hbs +375 -2
- package/views/config/service.hbs +35 -0
- package/workers/api.js +124 -45
- package/workers/documents.js +1 -0
- package/workers/export.js +926 -0
- package/workers/imap.js +29 -0
- package/workers/submit.js +25 -5
- package/workers/webhooks.js +11 -2
|
@@ -147,7 +147,8 @@ async function init(args) {
|
|
|
147
147
|
.lowercase()
|
|
148
148
|
.valid('html', 'plain', '*')
|
|
149
149
|
.example('*')
|
|
150
|
-
.description('Which text content to return, use * for all. By default text content is not returned.')
|
|
150
|
+
.description('Which text content to return, use * for all. By default text content is not returned.')
|
|
151
|
+
.label('MessageTextType'),
|
|
151
152
|
|
|
152
153
|
webSafeHtml: Joi.boolean()
|
|
153
154
|
.truthy('Y', 'true', '1')
|
|
@@ -310,9 +311,9 @@ async function init(args) {
|
|
|
310
311
|
}),
|
|
311
312
|
|
|
312
313
|
contentType: Joi.string().lowercase().max(256).example('image/gif'),
|
|
313
|
-
contentDisposition: Joi.string().lowercase().valid('inline', 'attachment'),
|
|
314
|
+
contentDisposition: Joi.string().lowercase().valid('inline', 'attachment').label('MsgUploadContentDisposition'),
|
|
314
315
|
cid: Joi.string().max(256).example('unique-image-id@localhost').description('Content-ID value for embedded images'),
|
|
315
|
-
encoding: Joi.string().valid('base64').default('base64'),
|
|
316
|
+
encoding: Joi.string().valid('base64').default('base64').label('MsgUploadEncoding'),
|
|
316
317
|
|
|
317
318
|
reference: Joi.string()
|
|
318
319
|
.base64({ paddingRequired: false, urlSafe: true })
|
|
@@ -322,6 +323,7 @@ async function init(args) {
|
|
|
322
323
|
.description(
|
|
323
324
|
'References an existing attachment by its ID instead of providing new attachment content. If this field is set, the `content` field must not be included. If not set, the `content` field is required.'
|
|
324
325
|
)
|
|
326
|
+
.label('MsgUploadReference')
|
|
325
327
|
}).label('UploadAttachment')
|
|
326
328
|
)
|
|
327
329
|
.description('List of attachments')
|
|
@@ -430,15 +432,15 @@ async function init(args) {
|
|
|
430
432
|
response: {
|
|
431
433
|
schema: Joi.object({
|
|
432
434
|
flags: Joi.object({
|
|
433
|
-
add: Joi.array().items(Joi.string()).example(['\\Seen', '\\Flagged']),
|
|
434
|
-
delete: Joi.array().items(Joi.string()).example(['\\Draft']),
|
|
435
|
-
set: Joi.array().items(Joi.string()).example(['\\Seen'])
|
|
436
|
-
}).label('
|
|
435
|
+
add: Joi.array().items(Joi.string().label('FlagEntry')).example(['\\Seen', '\\Flagged']).label('FlagAddList'),
|
|
436
|
+
delete: Joi.array().items(Joi.string().label('FlagEntry')).example(['\\Draft']).label('FlagDeleteList'),
|
|
437
|
+
set: Joi.array().items(Joi.string().label('FlagEntry')).example(['\\Seen']).label('FlagSetList')
|
|
438
|
+
}).label('FlagUpdateResponse'),
|
|
437
439
|
labels: Joi.object({
|
|
438
|
-
add: Joi.array().items(Joi.string()).example(['Label1', 'Label2']),
|
|
439
|
-
delete: Joi.array().items(Joi.string()).example(['Label3']),
|
|
440
|
-
set: Joi.array().items(Joi.string()).example(['Label1'])
|
|
441
|
-
}).label('
|
|
440
|
+
add: Joi.array().items(Joi.string().label('LabelEntry')).example(['Label1', 'Label2']).label('LabelAddList'),
|
|
441
|
+
delete: Joi.array().items(Joi.string().label('LabelEntry')).example(['Label3']).label('LabelDeleteList'),
|
|
442
|
+
set: Joi.array().items(Joi.string().label('LabelEntry')).example(['Label1']).label('LabelSetList')
|
|
443
|
+
}).label('LabelUpdateResponse')
|
|
442
444
|
}).label('MessageUpdateResponse'),
|
|
443
445
|
failAction: 'log'
|
|
444
446
|
}
|
|
@@ -510,16 +512,16 @@ async function init(args) {
|
|
|
510
512
|
response: {
|
|
511
513
|
schema: Joi.object({
|
|
512
514
|
flags: Joi.object({
|
|
513
|
-
add: Joi.array().items(Joi.string()).example(['\\Seen', '\\Flagged']),
|
|
514
|
-
delete: Joi.array().items(Joi.string()).example(['\\Draft']),
|
|
515
|
-
set: Joi.array().items(Joi.string()).example(['\\Seen'])
|
|
516
|
-
}).label('
|
|
515
|
+
add: Joi.array().items(Joi.string().label('BulkFlagEntry')).example(['\\Seen', '\\Flagged']).label('BulkFlagAddList'),
|
|
516
|
+
delete: Joi.array().items(Joi.string().label('BulkFlagEntry')).example(['\\Draft']).label('BulkFlagDeleteList'),
|
|
517
|
+
set: Joi.array().items(Joi.string().label('BulkFlagEntry')).example(['\\Seen']).label('BulkFlagSetList')
|
|
518
|
+
}).label('BulkFlagUpdateResponse'),
|
|
517
519
|
labels: Joi.object({
|
|
518
|
-
add: Joi.array().items(Joi.string()).example(['Label1', 'Label2']),
|
|
519
|
-
delete: Joi.array().items(Joi.string()).example(['Label3']),
|
|
520
|
-
set: Joi.array().items(Joi.string()).example(['Label1'])
|
|
521
|
-
}).label('
|
|
522
|
-
}).label('
|
|
520
|
+
add: Joi.array().items(Joi.string().label('BulkLabelEntry')).example(['Label1', 'Label2']).label('BulkLabelAddList'),
|
|
521
|
+
delete: Joi.array().items(Joi.string().label('BulkLabelEntry')).example(['Label3']).label('BulkLabelDeleteList'),
|
|
522
|
+
set: Joi.array().items(Joi.string().label('BulkLabelEntry')).example(['Label1']).label('BulkLabelSetList')
|
|
523
|
+
}).label('BulkLabelUpdateResponse')
|
|
524
|
+
}).label('BulkMessageUpdateResponse'),
|
|
523
525
|
failAction: 'log'
|
|
524
526
|
}
|
|
525
527
|
}
|
|
@@ -759,7 +761,9 @@ async function init(args) {
|
|
|
759
761
|
moved: Joi.object({
|
|
760
762
|
destination: Joi.string().required().example('Trash').description('Trash folder path').label('TrashPath'),
|
|
761
763
|
message: Joi.string().required().example('AAAAAwAAAWg').description('Message ID in Trash').label('TrashMessageId')
|
|
762
|
-
})
|
|
764
|
+
})
|
|
765
|
+
.description('Present if message was moved to Trash')
|
|
766
|
+
.label('MessageMovedToTrash')
|
|
763
767
|
}).label('MessageDeleteResponse'),
|
|
764
768
|
failAction: 'log'
|
|
765
769
|
}
|
|
@@ -1083,7 +1087,7 @@ async function init(args) {
|
|
|
1083
1087
|
.unknown()
|
|
1084
1088
|
.meta({ swaggerHidden: true })
|
|
1085
1089
|
})
|
|
1086
|
-
.label('
|
|
1090
|
+
.label('MessageSearchPayload')
|
|
1087
1091
|
.example({
|
|
1088
1092
|
search: {
|
|
1089
1093
|
unseen: true,
|
|
@@ -1281,7 +1285,8 @@ async function init(args) {
|
|
|
1281
1285
|
.valid('html', 'plain', '*')
|
|
1282
1286
|
.default('*')
|
|
1283
1287
|
.example('*')
|
|
1284
|
-
.description('Which text content to return, use * for all. By default all contents are returned.')
|
|
1288
|
+
.description('Which text content to return, use * for all. By default all contents are returned.')
|
|
1289
|
+
.label('TextSearchTextType'),
|
|
1285
1290
|
documentStore: documentStoreSchema.default(false)
|
|
1286
1291
|
}),
|
|
1287
1292
|
|
|
@@ -93,7 +93,11 @@ async function init(args) {
|
|
|
93
93
|
.example('Something about the template')
|
|
94
94
|
.description('Optional description of the template')
|
|
95
95
|
.label('TemplateDescription'),
|
|
96
|
-
format: Joi.string()
|
|
96
|
+
format: Joi.string()
|
|
97
|
+
.valid('html', 'markdown')
|
|
98
|
+
.default('html')
|
|
99
|
+
.description('Markup language for HTML ("html" or "markdown")')
|
|
100
|
+
.label('TemplateFormat'),
|
|
97
101
|
content: Joi.object({
|
|
98
102
|
subject: templateSchemas.subject,
|
|
99
103
|
text: templateSchemas.text,
|
|
@@ -176,7 +180,12 @@ async function init(args) {
|
|
|
176
180
|
.example('Something about the template')
|
|
177
181
|
.description('Optional description of the template')
|
|
178
182
|
.label('TemplateDescription'),
|
|
179
|
-
format: Joi.string()
|
|
183
|
+
format: Joi.string()
|
|
184
|
+
.empty('')
|
|
185
|
+
.valid('html', 'markdown')
|
|
186
|
+
.default('html')
|
|
187
|
+
.description('Markup language for HTML ("html" or "markdown")')
|
|
188
|
+
.label('TemplateUpdateFormat'),
|
|
180
189
|
content: Joi.object({
|
|
181
190
|
subject: templateSchemas.subject,
|
|
182
191
|
text: templateSchemas.text,
|
|
@@ -277,7 +286,11 @@ async function init(args) {
|
|
|
277
286
|
.example('Something about the template')
|
|
278
287
|
.description('Optional description of the template')
|
|
279
288
|
.label('TemplateDescription'),
|
|
280
|
-
format: Joi.string()
|
|
289
|
+
format: Joi.string()
|
|
290
|
+
.valid('html', 'markdown')
|
|
291
|
+
.default('html')
|
|
292
|
+
.description('Markup language for HTML ("html" or "markdown")')
|
|
293
|
+
.label('TemplateListFormat'),
|
|
281
294
|
created: Joi.date().iso().example('2021-02-17T13:43:18.860Z').description('The time this template was created'),
|
|
282
295
|
updated: Joi.date().iso().example('2021-02-17T13:43:18.860Z').description('The time this template was last updated')
|
|
283
296
|
}).label('AccountTemplate')
|
|
@@ -345,7 +358,11 @@ async function init(args) {
|
|
|
345
358
|
.example('Something about the template')
|
|
346
359
|
.description('Optional description of the template')
|
|
347
360
|
.label('TemplateDescription'),
|
|
348
|
-
format: Joi.string()
|
|
361
|
+
format: Joi.string()
|
|
362
|
+
.valid('html', 'markdown')
|
|
363
|
+
.default('html')
|
|
364
|
+
.description('Markup language for HTML ("html" or "markdown")')
|
|
365
|
+
.label('TemplateResponseFormat'),
|
|
349
366
|
created: Joi.date().iso().example('2021-02-17T13:43:18.860Z').description('The time this template was created'),
|
|
350
367
|
updated: Joi.date().iso().example('2021-02-17T13:43:18.860Z').description('The time this template was last updated'),
|
|
351
368
|
content: Joi.object({
|
|
@@ -353,7 +370,11 @@ async function init(args) {
|
|
|
353
370
|
text: templateSchemas.text,
|
|
354
371
|
html: templateSchemas.html,
|
|
355
372
|
previewText: templateSchemas.previewText,
|
|
356
|
-
format: Joi.string()
|
|
373
|
+
format: Joi.string()
|
|
374
|
+
.valid('html', 'markdown')
|
|
375
|
+
.default('html')
|
|
376
|
+
.description('Markup language for HTML ("html" or "markdown")')
|
|
377
|
+
.label('TemplateContentFormat')
|
|
357
378
|
}).label('RequestTemplateContent')
|
|
358
379
|
}).label('AccountTemplateResponse'),
|
|
359
380
|
failAction: 'log'
|
|
@@ -411,7 +432,7 @@ async function init(args) {
|
|
|
411
432
|
deleted: Joi.boolean().truthy('Y', 'true', '1').falsy('N', 'false', 0).default(true).description('Was the template deleted'),
|
|
412
433
|
account: accountIdSchema.required(),
|
|
413
434
|
id: Joi.string().max(256).required().example('AAABgS-UcAYAAAABAA').description('Template ID')
|
|
414
|
-
}).label('
|
|
435
|
+
}).label('DeleteTemplateResponse'),
|
|
415
436
|
failAction: 'log'
|
|
416
437
|
}
|
|
417
438
|
}
|
|
@@ -480,7 +501,7 @@ async function init(args) {
|
|
|
480
501
|
schema: Joi.object({
|
|
481
502
|
deleted: Joi.boolean().truthy('Y', 'true', '1').falsy('N', 'false', 0).default(true).description('Were the templates flushed'),
|
|
482
503
|
account: accountIdSchema.required()
|
|
483
|
-
}).label('
|
|
504
|
+
}).label('FlushTemplatesResponse'),
|
|
484
505
|
failAction: 'log'
|
|
485
506
|
}
|
|
486
507
|
}
|
package/lib/arf-detect.js
CHANGED
|
@@ -27,7 +27,7 @@ const arfDetect = async messageInfo => {
|
|
|
27
27
|
|
|
28
28
|
let returnPath;
|
|
29
29
|
|
|
30
|
-
for (let attachment of messageInfo.attachments) {
|
|
30
|
+
for (let attachment of messageInfo.attachments || []) {
|
|
31
31
|
switch (attachment.contentType.toLowerCase()) {
|
|
32
32
|
case 'message/feedback-report': {
|
|
33
33
|
// found a feedback report
|
package/lib/consts.js
CHANGED
|
@@ -87,6 +87,22 @@ module.exports = {
|
|
|
87
87
|
LIST_SUBSCRIBE_NOTIFY: 'listSubscribe',
|
|
88
88
|
LIST_SUBSCRIBE_DESCRIPTION: 'Recipient Subscribed - A recipient re-subscribed to a list',
|
|
89
89
|
|
|
90
|
+
EXPORT_COMPLETED_NOTIFY: 'exportCompleted',
|
|
91
|
+
EXPORT_COMPLETED_DESCRIPTION: 'Export Completed - A bulk message export has completed successfully',
|
|
92
|
+
|
|
93
|
+
EXPORT_FAILED_NOTIFY: 'exportFailed',
|
|
94
|
+
EXPORT_FAILED_DESCRIPTION: 'Export Failed - A bulk message export has failed',
|
|
95
|
+
|
|
96
|
+
// Export defaults
|
|
97
|
+
DEFAULT_EXPORT_MAX_AGE: 24 * 60 * 60 * 1000, // 24 hours
|
|
98
|
+
DEFAULT_EXPORT_MAX_SIZE: 10 * 1024 * 1024 * 1024, // 10GB
|
|
99
|
+
DEFAULT_EXPORT_MAX_MESSAGES: 500000,
|
|
100
|
+
DEFAULT_EXPORT_MAX_CONCURRENT: 2,
|
|
101
|
+
DEFAULT_EXPORT_MAX_GLOBAL_CONCURRENT: 8,
|
|
102
|
+
DEFAULT_EXPORT_MAX_MESSAGE_SIZE: 50 * 1024 * 1024, // 50MB
|
|
103
|
+
DEFAULT_GMAIL_EXPORT_BATCH_SIZE: 10,
|
|
104
|
+
DEFAULT_OUTLOOK_EXPORT_BATCH_SIZE: 20,
|
|
105
|
+
|
|
90
106
|
MAX_DAYS_STATS: 7,
|
|
91
107
|
|
|
92
108
|
DEFAULT_MAX_LOG_LINES: 10000,
|
package/lib/db.js
CHANGED
|
@@ -107,12 +107,14 @@ const reqisQueue = new Redis(REDIS_CONF);
|
|
|
107
107
|
|
|
108
108
|
module.exports.queueConf = {
|
|
109
109
|
connection: reqisQueue,
|
|
110
|
+
sharedConnection: true, // Prevent BullMQ from closing shared connection
|
|
110
111
|
prefix: `${REDIS_PREFIX}bull`
|
|
111
112
|
};
|
|
112
113
|
|
|
113
114
|
const notifyQueue = new Queue('notify', module.exports.queueConf);
|
|
114
115
|
const submitQueue = new Queue('submit', module.exports.queueConf);
|
|
115
116
|
const documentsQueue = new Queue('documents', module.exports.queueConf);
|
|
117
|
+
const exportQueue = new Queue('export', module.exports.queueConf);
|
|
116
118
|
|
|
117
119
|
const zExpungeScript = fs.readFileSync(pathlib.join(__dirname, '/lua/z-expunge.lua'), 'utf-8');
|
|
118
120
|
const zSetScript = fs.readFileSync(pathlib.join(__dirname, 'lua/z-set.lua'), 'utf-8');
|
|
@@ -215,6 +217,7 @@ module.exports.redis = redis;
|
|
|
215
217
|
module.exports.notifyQueue = notifyQueue;
|
|
216
218
|
module.exports.submitQueue = submitQueue;
|
|
217
219
|
module.exports.documentsQueue = documentsQueue;
|
|
220
|
+
module.exports.exportQueue = exportQueue;
|
|
218
221
|
|
|
219
222
|
// do not set up the flow producer by default
|
|
220
223
|
module.exports.getFlowProducer = () => new FlowProducer(module.exports.queueConf /*, Redis*/);
|
|
@@ -1984,7 +1984,7 @@ class BaseClient {
|
|
|
1984
1984
|
// Store message in Redis
|
|
1985
1985
|
await this.redis.hsetBuffer(`${REDIS_PREFIX}iaq:${this.account}`, queueId, msgEntry);
|
|
1986
1986
|
|
|
1987
|
-
let queueKeep = (await settings.get('queueKeep'))
|
|
1987
|
+
let queueKeep = (await settings.get('queueKeep')) ?? true;
|
|
1988
1988
|
|
|
1989
1989
|
// Configure delivery retry settings
|
|
1990
1990
|
let defaultDeliveryAttempts = await settings.get('deliveryAttempts');
|
|
@@ -2007,14 +2007,16 @@ class BaseClient {
|
|
|
2007
2007
|
|
|
2008
2008
|
// Configure queue job options
|
|
2009
2009
|
let queueName = 'queued';
|
|
2010
|
+
let retention = typeof queueKeep === 'number' ? { age: 24 * 3600, count: queueKeep } : queueKeep;
|
|
2010
2011
|
let jobOpts = {
|
|
2011
2012
|
jobId: queueId,
|
|
2012
|
-
removeOnComplete:
|
|
2013
|
-
removeOnFail:
|
|
2013
|
+
removeOnComplete: retention,
|
|
2014
|
+
removeOnFail: retention,
|
|
2014
2015
|
attempts: typeof deliveryAttempts === 'number' ? deliveryAttempts : defaultDeliveryAttempts,
|
|
2015
2016
|
backoff: {
|
|
2016
2017
|
type: 'exponential',
|
|
2017
|
-
delay: 5000
|
|
2018
|
+
delay: 5000,
|
|
2019
|
+
jitter: 0.2 // 20% randomization to prevent thundering herd
|
|
2018
2020
|
}
|
|
2019
2021
|
};
|
|
2020
2022
|
|
|
@@ -11,6 +11,7 @@ const he = require('he');
|
|
|
11
11
|
const { BaseClient, metricsMeta } = require('./base-client');
|
|
12
12
|
const { mimeHtml } = require('@postalsys/email-text-tools');
|
|
13
13
|
const { emitChangeEvent } = require('../tools');
|
|
14
|
+
const crypto = require('crypto');
|
|
14
15
|
const { Gateway } = require('../gateway');
|
|
15
16
|
|
|
16
17
|
const {
|
|
@@ -20,11 +21,16 @@ const {
|
|
|
20
21
|
EMAIL_SENT_NOTIFY,
|
|
21
22
|
REDIS_PREFIX,
|
|
22
23
|
AUTH_ERROR_NOTIFY,
|
|
23
|
-
AUTH_SUCCESS_NOTIFY
|
|
24
|
+
AUTH_SUCCESS_NOTIFY,
|
|
25
|
+
DEFAULT_GMAIL_EXPORT_BATCH_SIZE
|
|
24
26
|
} = require('../consts');
|
|
25
27
|
|
|
28
|
+
const settings = require('../settings');
|
|
29
|
+
|
|
26
30
|
const { GMAIL_API_BASE, LIST_BATCH_SIZE, request: gmailApiRequest } = require('./gmail/gmail-api');
|
|
27
31
|
|
|
32
|
+
const MAX_GMAIL_BATCH_SIZE = 50;
|
|
33
|
+
|
|
28
34
|
// Labels to exclude from folder listings
|
|
29
35
|
const SKIP_LABELS = ['UNREAD', 'STARRED', 'IMPORTANT', 'CHAT', 'CATEGORY_PERSONAL'];
|
|
30
36
|
|
|
@@ -602,23 +608,39 @@ class GmailClient extends BaseClient {
|
|
|
602
608
|
|
|
603
609
|
return folderData;
|
|
604
610
|
})
|
|
605
|
-
.filter(value => value)
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
611
|
+
.filter(value => value);
|
|
612
|
+
|
|
613
|
+
// Add virtual "All Mail" folder for Gmail API
|
|
614
|
+
// This allows exporting all messages without scanning individual labels
|
|
615
|
+
mailboxes.unshift({
|
|
616
|
+
id: 'virtual_all',
|
|
617
|
+
path: '\\All',
|
|
618
|
+
delimiter: '/',
|
|
619
|
+
parentPath: '',
|
|
620
|
+
name: 'All Mail',
|
|
621
|
+
listed: true,
|
|
622
|
+
subscribed: false,
|
|
623
|
+
noSelect: true,
|
|
624
|
+
specialUse: '\\All',
|
|
625
|
+
specialUseSource: 'extension'
|
|
626
|
+
});
|
|
613
627
|
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
628
|
+
mailboxes.sort((a, b) => {
|
|
629
|
+
// Sort: INBOX first, then special folders, then alphabetical
|
|
630
|
+
if (a.path === 'INBOX') {
|
|
631
|
+
return -1;
|
|
632
|
+
} else if (b.path === 'INBOX') {
|
|
633
|
+
return 1;
|
|
634
|
+
}
|
|
619
635
|
|
|
620
|
-
|
|
621
|
-
|
|
636
|
+
if (a.specialUse && !b.specialUse) {
|
|
637
|
+
return -1;
|
|
638
|
+
} else if (!a.specialUse && b.specialUse) {
|
|
639
|
+
return 1;
|
|
640
|
+
}
|
|
641
|
+
|
|
642
|
+
return a.path.toLowerCase().localeCompare(b.path.toLowerCase());
|
|
643
|
+
});
|
|
622
644
|
|
|
623
645
|
return mailboxes;
|
|
624
646
|
}
|
|
@@ -756,7 +778,7 @@ class GmailClient extends BaseClient {
|
|
|
756
778
|
let nextPageCursor = pageCursor.nextPageCursor(listingResult.nextPageToken);
|
|
757
779
|
let prevPageCursor = pageCursor.prevPageCursor();
|
|
758
780
|
|
|
759
|
-
if (options.metadataOnly) {
|
|
781
|
+
if (options.metadataOnly || query.metadataOnly) {
|
|
760
782
|
// Return just IDs without fetching full content
|
|
761
783
|
return {
|
|
762
784
|
total: messageCount,
|
|
@@ -769,6 +791,8 @@ class GmailClient extends BaseClient {
|
|
|
769
791
|
}
|
|
770
792
|
|
|
771
793
|
// Fetch message content for matching messages in batches
|
|
794
|
+
// Use format=minimal for minimalFields option (faster, returns only id, threadId, labelIds, internalDate, sizeEstimate)
|
|
795
|
+
const messageFormat = options.minimalFields ? 'minimal' : undefined;
|
|
772
796
|
|
|
773
797
|
let promises = [];
|
|
774
798
|
|
|
@@ -782,14 +806,16 @@ class GmailClient extends BaseClient {
|
|
|
782
806
|
throw entry.reason;
|
|
783
807
|
}
|
|
784
808
|
if (entry.value) {
|
|
785
|
-
messageList.push(this.formatMessage(entry.value, { path }));
|
|
809
|
+
messageList.push(this.formatMessage(entry.value, { path, minimalFields: options.minimalFields }));
|
|
786
810
|
}
|
|
787
811
|
}
|
|
788
812
|
promises = [];
|
|
789
813
|
};
|
|
790
814
|
|
|
791
815
|
for (let { id: message } of listingResult.messages || []) {
|
|
792
|
-
|
|
816
|
+
let requestUrl = `${GMAIL_API_BASE}/gmail/v1/users/me/messages/${message}`;
|
|
817
|
+
let requestParams = messageFormat ? { format: messageFormat } : undefined;
|
|
818
|
+
promises.push(this.request(requestUrl, 'get', requestParams));
|
|
793
819
|
if (promises.length > LIST_BATCH_SIZE) {
|
|
794
820
|
await resolvePromises();
|
|
795
821
|
}
|
|
@@ -1269,6 +1295,9 @@ class GmailClient extends BaseClient {
|
|
|
1269
1295
|
|
|
1270
1296
|
let formattedMessage = this.formatMessage(messageData, { extended: true, textType: options.textType });
|
|
1271
1297
|
|
|
1298
|
+
// Resolve label IDs to human-readable names
|
|
1299
|
+
await this.resolveLabels(formattedMessage);
|
|
1300
|
+
|
|
1272
1301
|
// Mark as seen if requested
|
|
1273
1302
|
if (options.markAsSeen && (!formattedMessage.flags || !formattedMessage.flags.includes('\\Seen'))) {
|
|
1274
1303
|
//
|
|
@@ -1325,6 +1354,64 @@ class GmailClient extends BaseClient {
|
|
|
1325
1354
|
return formattedMessage;
|
|
1326
1355
|
}
|
|
1327
1356
|
|
|
1357
|
+
/**
|
|
1358
|
+
* Fetches multiple messages in parallel for batch export operations
|
|
1359
|
+
* @param {string[]} emailIds - Array of message IDs
|
|
1360
|
+
* @param {Object} options - Fetch options
|
|
1361
|
+
* @returns {Object[]} Array of results with messageId, data, and error fields
|
|
1362
|
+
*/
|
|
1363
|
+
async getMessages(emailIds, options) {
|
|
1364
|
+
options = options || {};
|
|
1365
|
+
await this.prepare();
|
|
1366
|
+
|
|
1367
|
+
// Pre-fetch labels to resolve label IDs to names
|
|
1368
|
+
const labelMap = new Map();
|
|
1369
|
+
try {
|
|
1370
|
+
const labelsResult = await this.getLabels();
|
|
1371
|
+
for (const label of labelsResult || []) {
|
|
1372
|
+
labelMap.set(label.id, label.name);
|
|
1373
|
+
}
|
|
1374
|
+
} catch (err) {
|
|
1375
|
+
this.logger.warn({ msg: 'Failed to fetch labels for export, using raw label IDs', account: this.account, err });
|
|
1376
|
+
}
|
|
1377
|
+
|
|
1378
|
+
const results = [];
|
|
1379
|
+
const settingsBatchSize = await settings.get('gmailExportBatchSize');
|
|
1380
|
+
const batchSize = Math.min(settingsBatchSize || DEFAULT_GMAIL_EXPORT_BATCH_SIZE, MAX_GMAIL_BATCH_SIZE);
|
|
1381
|
+
|
|
1382
|
+
for (let i = 0; i < emailIds.length; i += batchSize) {
|
|
1383
|
+
const batch = emailIds.slice(i, i + batchSize);
|
|
1384
|
+
|
|
1385
|
+
const batchResults = await Promise.all(
|
|
1386
|
+
batch.map(async emailId => {
|
|
1387
|
+
try {
|
|
1388
|
+
const requestQuery = { format: 'full' };
|
|
1389
|
+
const messageData = await this.request(`${GMAIL_API_BASE}/gmail/v1/users/me/messages/${emailId}`, 'get', requestQuery);
|
|
1390
|
+
const formattedMessage = this.formatMessage(messageData, { extended: true, textType: options.textType });
|
|
1391
|
+
|
|
1392
|
+
await this.resolveLabels(formattedMessage, labelMap);
|
|
1393
|
+
|
|
1394
|
+
return {
|
|
1395
|
+
messageId: emailId,
|
|
1396
|
+
data: formattedMessage,
|
|
1397
|
+
error: null
|
|
1398
|
+
};
|
|
1399
|
+
} catch (err) {
|
|
1400
|
+
return {
|
|
1401
|
+
messageId: emailId,
|
|
1402
|
+
data: null,
|
|
1403
|
+
error: { message: err.message, code: err.code, statusCode: err.statusCode }
|
|
1404
|
+
};
|
|
1405
|
+
}
|
|
1406
|
+
})
|
|
1407
|
+
);
|
|
1408
|
+
|
|
1409
|
+
results.push(...batchResults);
|
|
1410
|
+
}
|
|
1411
|
+
|
|
1412
|
+
return results;
|
|
1413
|
+
}
|
|
1414
|
+
|
|
1328
1415
|
/**
|
|
1329
1416
|
* Fetches text content for a message
|
|
1330
1417
|
* @param {string} textId - Encoded text identifier
|
|
@@ -1521,19 +1608,45 @@ class GmailClient extends BaseClient {
|
|
|
1521
1608
|
return false;
|
|
1522
1609
|
}
|
|
1523
1610
|
|
|
1524
|
-
//
|
|
1525
|
-
|
|
1526
|
-
let
|
|
1527
|
-
let
|
|
1611
|
+
// Gmail JSON endpoint: 5MB body limit (~3.5MB raw before base64url overhead)
|
|
1612
|
+
// Gmail upload endpoint: 35MB raw RFC822
|
|
1613
|
+
let contentType;
|
|
1614
|
+
let payload;
|
|
1615
|
+
let targetEndpoint;
|
|
1616
|
+
const JSON_SEND_LIMIT = 3.5 * 1024 * 1024;
|
|
1528
1617
|
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
targetEndpoint = `/gmail/v1/users/me/messages/send`;
|
|
1618
|
+
if (raw.length <= JSON_SEND_LIMIT) {
|
|
1619
|
+
// JSON endpoint with base64url encoding (retry-safe, no ArrayBuffer issues)
|
|
1532
1620
|
contentType = 'application/json';
|
|
1533
|
-
payload = {
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1621
|
+
payload = { raw: raw.toString('base64url') };
|
|
1622
|
+
targetEndpoint = `/gmail/v1/users/me/messages/send`;
|
|
1623
|
+
if (data?.reference?.threadId) {
|
|
1624
|
+
payload.threadId = data.reference.threadId;
|
|
1625
|
+
}
|
|
1626
|
+
} else if (data?.reference?.threadId) {
|
|
1627
|
+
// Large threaded reply: multipart upload preserves explicit threadId
|
|
1628
|
+
// via JSON metadata alongside the raw RFC822 message body
|
|
1629
|
+
const boundary = `ee_${crypto.randomBytes(16).toString('hex')}`;
|
|
1630
|
+
const metadata = JSON.stringify({ threadId: data.reference.threadId });
|
|
1631
|
+
const preamble = Buffer.from(
|
|
1632
|
+
`--${boundary}\r\n` +
|
|
1633
|
+
`Content-Type: application/json; charset=UTF-8\r\n` +
|
|
1634
|
+
`\r\n` +
|
|
1635
|
+
`${metadata}\r\n` +
|
|
1636
|
+
`--${boundary}\r\n` +
|
|
1637
|
+
`Content-Type: message/rfc822\r\n` +
|
|
1638
|
+
`\r\n`
|
|
1639
|
+
);
|
|
1640
|
+
const epilogue = Buffer.from(`\r\n--${boundary}--`);
|
|
1641
|
+
|
|
1642
|
+
contentType = `multipart/related; boundary=${boundary}`;
|
|
1643
|
+
payload = Buffer.concat([preamble, raw, epilogue]);
|
|
1644
|
+
targetEndpoint = `/upload/gmail/v1/users/me/messages/send?uploadType=multipart`;
|
|
1645
|
+
} else {
|
|
1646
|
+
// Large non-threaded message: simple upload with raw RFC822 Buffer
|
|
1647
|
+
contentType = 'message/rfc822';
|
|
1648
|
+
payload = raw;
|
|
1649
|
+
targetEndpoint = `/upload/gmail/v1/users/me/messages/send`;
|
|
1537
1650
|
}
|
|
1538
1651
|
|
|
1539
1652
|
// Send via Gmail API
|
|
@@ -1591,7 +1704,8 @@ class GmailClient extends BaseClient {
|
|
|
1591
1704
|
await this.redis
|
|
1592
1705
|
.multi()
|
|
1593
1706
|
.hset(data.feedbackKey, 'success', 'true')
|
|
1594
|
-
.expire(1 * 60 * 60)
|
|
1707
|
+
.expire(data.feedbackKey, 1 * 60 * 60)
|
|
1708
|
+
.exec();
|
|
1595
1709
|
}
|
|
1596
1710
|
|
|
1597
1711
|
return {
|
|
@@ -2042,6 +2156,35 @@ class GmailClient extends BaseClient {
|
|
|
2042
2156
|
}
|
|
2043
2157
|
}
|
|
2044
2158
|
|
|
2159
|
+
/**
|
|
2160
|
+
* Resolves label IDs to human-readable names on a formatted message.
|
|
2161
|
+
* Mutates formattedMessage.labels in place. Labels starting with '\\' are
|
|
2162
|
+
* treated as special-use labels and left as-is.
|
|
2163
|
+
*
|
|
2164
|
+
* @param {Object} formattedMessage - Message with a .labels array
|
|
2165
|
+
* @param {Map} [labelMap] - Optional pre-built id-to-name map (avoids an extra getLabels call)
|
|
2166
|
+
*/
|
|
2167
|
+
async resolveLabels(formattedMessage, labelMap) {
|
|
2168
|
+
if (!Array.isArray(formattedMessage?.labels)) {
|
|
2169
|
+
return;
|
|
2170
|
+
}
|
|
2171
|
+
|
|
2172
|
+
if (!labelMap) {
|
|
2173
|
+
const labelsResult = await this.getLabels();
|
|
2174
|
+
labelMap = new Map();
|
|
2175
|
+
for (const label of labelsResult || []) {
|
|
2176
|
+
labelMap.set(label.id, label.name);
|
|
2177
|
+
}
|
|
2178
|
+
}
|
|
2179
|
+
|
|
2180
|
+
formattedMessage.labels = formattedMessage.labels.map(label => {
|
|
2181
|
+
if (label.startsWith('\\')) {
|
|
2182
|
+
return label;
|
|
2183
|
+
}
|
|
2184
|
+
return labelMap.get(label) || label;
|
|
2185
|
+
});
|
|
2186
|
+
}
|
|
2187
|
+
|
|
2045
2188
|
/**
|
|
2046
2189
|
* Resolves a label by path or ID
|
|
2047
2190
|
* @param {string} path - Label path or ID
|
|
@@ -2292,7 +2435,7 @@ class GmailClient extends BaseClient {
|
|
|
2292
2435
|
* @returns {Object} Formatted message
|
|
2293
2436
|
*/
|
|
2294
2437
|
formatMessage(messageData, options) {
|
|
2295
|
-
let { extended, path, textType } = options || {};
|
|
2438
|
+
let { extended, path, textType, minimalFields } = options || {};
|
|
2296
2439
|
|
|
2297
2440
|
let date = messageData.internalDate && !isNaN(messageData.internalDate) ? new Date(Number(messageData.internalDate)) : undefined;
|
|
2298
2441
|
if (date?.toString() === 'Invalid Date') {
|
|
@@ -2301,6 +2444,34 @@ class GmailClient extends BaseClient {
|
|
|
2301
2444
|
|
|
2302
2445
|
let { flags, labels, category } = this.formatFlagsAndLabels(messageData);
|
|
2303
2446
|
|
|
2447
|
+
// For minimalFields mode (format=minimal), payload is not available
|
|
2448
|
+
// Return only basic fields: id, threadId, labelIds, internalDate, sizeEstimate
|
|
2449
|
+
if (minimalFields) {
|
|
2450
|
+
const result = {
|
|
2451
|
+
id: messageData.id,
|
|
2452
|
+
emailId: messageData.id || undefined,
|
|
2453
|
+
threadId: messageData.threadId || undefined,
|
|
2454
|
+
date: date ? date.toISOString() : undefined,
|
|
2455
|
+
flags,
|
|
2456
|
+
labels,
|
|
2457
|
+
category,
|
|
2458
|
+
unseen: !flags.includes('\\Seen') ? true : undefined,
|
|
2459
|
+
flagged: flags.includes('\\Flagged') ? true : undefined,
|
|
2460
|
+
draft: flags.includes('\\Draft') ? true : undefined,
|
|
2461
|
+
size: messageData.sizeEstimate
|
|
2462
|
+
};
|
|
2463
|
+
|
|
2464
|
+
// Set special-use based on labels
|
|
2465
|
+
for (let specialUseTag of ['\\Junk', '\\Sent', '\\Trash', '\\Inbox', '\\Drafts']) {
|
|
2466
|
+
if (result.labels && result.labels.includes(specialUseTag)) {
|
|
2467
|
+
result.messageSpecialUse = specialUseTag;
|
|
2468
|
+
break;
|
|
2469
|
+
}
|
|
2470
|
+
}
|
|
2471
|
+
|
|
2472
|
+
return result;
|
|
2473
|
+
}
|
|
2474
|
+
|
|
2304
2475
|
let envelope = this.getEnvelope(messageData);
|
|
2305
2476
|
|
|
2306
2477
|
// Extract all headers
|
|
@@ -2424,7 +2595,7 @@ class GmailClient extends BaseClient {
|
|
|
2424
2595
|
* @returns {string} Formatted term
|
|
2425
2596
|
*/
|
|
2426
2597
|
formatSearchTerm(term, quot = '"') {
|
|
2427
|
-
if (typeof term === 'object' && term && Object.prototype.toString.apply(
|
|
2598
|
+
if (typeof term === 'object' && term && Object.prototype.toString.apply(term) === '[object Date]') {
|
|
2428
2599
|
term = term.toISOString().substring(0, 10);
|
|
2429
2600
|
}
|
|
2430
2601
|
|