emailengine-app 2.61.4 → 2.62.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/CHANGELOG.md +87 -0
  2. package/data/google-crawlers.json +1 -1
  3. package/lib/account.js +20 -7
  4. package/lib/api-routes/account-routes.js +28 -5
  5. package/lib/api-routes/chat-routes.js +1 -1
  6. package/lib/api-routes/export-routes.js +316 -0
  7. package/lib/api-routes/message-routes.js +28 -23
  8. package/lib/api-routes/template-routes.js +28 -7
  9. package/lib/arf-detect.js +1 -1
  10. package/lib/consts.js +16 -0
  11. package/lib/db.js +3 -0
  12. package/lib/email-client/base-client.js +6 -4
  13. package/lib/email-client/gmail-client.js +204 -33
  14. package/lib/email-client/imap/mailbox.js +99 -8
  15. package/lib/email-client/imap/subconnection.js +5 -5
  16. package/lib/email-client/imap-client.js +76 -16
  17. package/lib/email-client/message-builder.js +3 -1
  18. package/lib/email-client/notification-handler.js +12 -9
  19. package/lib/email-client/outlook-client.js +362 -69
  20. package/lib/email-client/smtp-pool-manager.js +1 -1
  21. package/lib/export.js +528 -0
  22. package/lib/oauth/gmail.js +21 -13
  23. package/lib/oauth/mail-ru.js +23 -10
  24. package/lib/oauth/outlook.js +26 -16
  25. package/lib/oauth/pubsub/google.js +5 -0
  26. package/lib/routes-ui.js +236 -2
  27. package/lib/schemas.js +260 -80
  28. package/lib/stream-encrypt.js +263 -0
  29. package/lib/tools.js +30 -4
  30. package/lib/ui-routes/account-routes.js +24 -1
  31. package/lib/ui-routes/admin-config-routes.js +11 -4
  32. package/lib/ui-routes/admin-entities-routes.js +18 -0
  33. package/lib/webhooks.js +16 -20
  34. package/package.json +17 -17
  35. package/sbom.json +1 -1
  36. package/server.js +41 -5
  37. package/static/js/ace/ace.js +1 -1
  38. package/static/js/ace/ext-language_tools.js +1 -1
  39. package/static/licenses.html +47 -127
  40. package/translations/de.mo +0 -0
  41. package/translations/de.po +63 -36
  42. package/translations/en.mo +0 -0
  43. package/translations/en.po +64 -37
  44. package/translations/et.mo +0 -0
  45. package/translations/et.po +63 -36
  46. package/translations/fr.mo +0 -0
  47. package/translations/fr.po +63 -36
  48. package/translations/ja.mo +0 -0
  49. package/translations/ja.po +63 -36
  50. package/translations/messages.pot +88 -55
  51. package/translations/nl.mo +0 -0
  52. package/translations/nl.po +63 -36
  53. package/translations/pl.mo +0 -0
  54. package/translations/pl.po +63 -36
  55. package/views/accounts/account.hbs +375 -2
  56. package/views/config/service.hbs +35 -0
  57. package/workers/api.js +124 -45
  58. package/workers/documents.js +1 -0
  59. package/workers/export.js +926 -0
  60. package/workers/imap.js +29 -0
  61. package/workers/submit.js +25 -5
  62. package/workers/webhooks.js +11 -2
@@ -147,7 +147,8 @@ async function init(args) {
147
147
  .lowercase()
148
148
  .valid('html', 'plain', '*')
149
149
  .example('*')
150
- .description('Which text content to return, use * for all. By default text content is not returned.'),
150
+ .description('Which text content to return, use * for all. By default text content is not returned.')
151
+ .label('MessageTextType'),
151
152
 
152
153
  webSafeHtml: Joi.boolean()
153
154
  .truthy('Y', 'true', '1')
@@ -310,9 +311,9 @@ async function init(args) {
310
311
  }),
311
312
 
312
313
  contentType: Joi.string().lowercase().max(256).example('image/gif'),
313
- contentDisposition: Joi.string().lowercase().valid('inline', 'attachment'),
314
+ contentDisposition: Joi.string().lowercase().valid('inline', 'attachment').label('MsgUploadContentDisposition'),
314
315
  cid: Joi.string().max(256).example('unique-image-id@localhost').description('Content-ID value for embedded images'),
315
- encoding: Joi.string().valid('base64').default('base64'),
316
+ encoding: Joi.string().valid('base64').default('base64').label('MsgUploadEncoding'),
316
317
 
317
318
  reference: Joi.string()
318
319
  .base64({ paddingRequired: false, urlSafe: true })
@@ -322,6 +323,7 @@ async function init(args) {
322
323
  .description(
323
324
  'References an existing attachment by its ID instead of providing new attachment content. If this field is set, the `content` field must not be included. If not set, the `content` field is required.'
324
325
  )
326
+ .label('MsgUploadReference')
325
327
  }).label('UploadAttachment')
326
328
  )
327
329
  .description('List of attachments')
@@ -430,15 +432,15 @@ async function init(args) {
430
432
  response: {
431
433
  schema: Joi.object({
432
434
  flags: Joi.object({
433
- add: Joi.array().items(Joi.string()).example(['\\Seen', '\\Flagged']),
434
- delete: Joi.array().items(Joi.string()).example(['\\Draft']),
435
- set: Joi.array().items(Joi.string()).example(['\\Seen'])
436
- }).label('FlagResponse'),
435
+ add: Joi.array().items(Joi.string().label('FlagEntry')).example(['\\Seen', '\\Flagged']).label('FlagAddList'),
436
+ delete: Joi.array().items(Joi.string().label('FlagEntry')).example(['\\Draft']).label('FlagDeleteList'),
437
+ set: Joi.array().items(Joi.string().label('FlagEntry')).example(['\\Seen']).label('FlagSetList')
438
+ }).label('FlagUpdateResponse'),
437
439
  labels: Joi.object({
438
- add: Joi.array().items(Joi.string()).example(['Label1', 'Label2']),
439
- delete: Joi.array().items(Joi.string()).example(['Label3']),
440
- set: Joi.array().items(Joi.string()).example(['Label1'])
441
- }).label('FlagResponse')
440
+ add: Joi.array().items(Joi.string().label('LabelEntry')).example(['Label1', 'Label2']).label('LabelAddList'),
441
+ delete: Joi.array().items(Joi.string().label('LabelEntry')).example(['Label3']).label('LabelDeleteList'),
442
+ set: Joi.array().items(Joi.string().label('LabelEntry')).example(['Label1']).label('LabelSetList')
443
+ }).label('LabelUpdateResponse')
442
444
  }).label('MessageUpdateResponse'),
443
445
  failAction: 'log'
444
446
  }
@@ -510,16 +512,16 @@ async function init(args) {
510
512
  response: {
511
513
  schema: Joi.object({
512
514
  flags: Joi.object({
513
- add: Joi.array().items(Joi.string()).example(['\\Seen', '\\Flagged']),
514
- delete: Joi.array().items(Joi.string()).example(['\\Draft']),
515
- set: Joi.array().items(Joi.string()).example(['\\Seen'])
516
- }).label('FlagResponse'),
515
+ add: Joi.array().items(Joi.string().label('BulkFlagEntry')).example(['\\Seen', '\\Flagged']).label('BulkFlagAddList'),
516
+ delete: Joi.array().items(Joi.string().label('BulkFlagEntry')).example(['\\Draft']).label('BulkFlagDeleteList'),
517
+ set: Joi.array().items(Joi.string().label('BulkFlagEntry')).example(['\\Seen']).label('BulkFlagSetList')
518
+ }).label('BulkFlagUpdateResponse'),
517
519
  labels: Joi.object({
518
- add: Joi.array().items(Joi.string()).example(['Label1', 'Label2']),
519
- delete: Joi.array().items(Joi.string()).example(['Label3']),
520
- set: Joi.array().items(Joi.string()).example(['Label1'])
521
- }).label('FlagResponse')
522
- }).label('MessageUpdateResponse'),
520
+ add: Joi.array().items(Joi.string().label('BulkLabelEntry')).example(['Label1', 'Label2']).label('BulkLabelAddList'),
521
+ delete: Joi.array().items(Joi.string().label('BulkLabelEntry')).example(['Label3']).label('BulkLabelDeleteList'),
522
+ set: Joi.array().items(Joi.string().label('BulkLabelEntry')).example(['Label1']).label('BulkLabelSetList')
523
+ }).label('BulkLabelUpdateResponse')
524
+ }).label('BulkMessageUpdateResponse'),
523
525
  failAction: 'log'
524
526
  }
525
527
  }
@@ -759,7 +761,9 @@ async function init(args) {
759
761
  moved: Joi.object({
760
762
  destination: Joi.string().required().example('Trash').description('Trash folder path').label('TrashPath'),
761
763
  message: Joi.string().required().example('AAAAAwAAAWg').description('Message ID in Trash').label('TrashMessageId')
762
- }).description('Present if message was moved to Trash')
764
+ })
765
+ .description('Present if message was moved to Trash')
766
+ .label('MessageMovedToTrash')
763
767
  }).label('MessageDeleteResponse'),
764
768
  failAction: 'log'
765
769
  }
@@ -1083,7 +1087,7 @@ async function init(args) {
1083
1087
  .unknown()
1084
1088
  .meta({ swaggerHidden: true })
1085
1089
  })
1086
- .label('SearchQuery')
1090
+ .label('MessageSearchPayload')
1087
1091
  .example({
1088
1092
  search: {
1089
1093
  unseen: true,
@@ -1281,7 +1285,8 @@ async function init(args) {
1281
1285
  .valid('html', 'plain', '*')
1282
1286
  .default('*')
1283
1287
  .example('*')
1284
- .description('Which text content to return, use * for all. By default all contents are returned.'),
1288
+ .description('Which text content to return, use * for all. By default all contents are returned.')
1289
+ .label('TextSearchTextType'),
1285
1290
  documentStore: documentStoreSchema.default(false)
1286
1291
  }),
1287
1292
 
@@ -93,7 +93,11 @@ async function init(args) {
93
93
  .example('Something about the template')
94
94
  .description('Optional description of the template')
95
95
  .label('TemplateDescription'),
96
- format: Joi.string().valid('html', 'markdown').default('html').description('Markup language for HTML ("html" or "markdown")'),
96
+ format: Joi.string()
97
+ .valid('html', 'markdown')
98
+ .default('html')
99
+ .description('Markup language for HTML ("html" or "markdown")')
100
+ .label('TemplateFormat'),
97
101
  content: Joi.object({
98
102
  subject: templateSchemas.subject,
99
103
  text: templateSchemas.text,
@@ -176,7 +180,12 @@ async function init(args) {
176
180
  .example('Something about the template')
177
181
  .description('Optional description of the template')
178
182
  .label('TemplateDescription'),
179
- format: Joi.string().empty('').valid('html', 'markdown').default('html').description('Markup language for HTML ("html" or "markdown")'),
183
+ format: Joi.string()
184
+ .empty('')
185
+ .valid('html', 'markdown')
186
+ .default('html')
187
+ .description('Markup language for HTML ("html" or "markdown")')
188
+ .label('TemplateUpdateFormat'),
180
189
  content: Joi.object({
181
190
  subject: templateSchemas.subject,
182
191
  text: templateSchemas.text,
@@ -277,7 +286,11 @@ async function init(args) {
277
286
  .example('Something about the template')
278
287
  .description('Optional description of the template')
279
288
  .label('TemplateDescription'),
280
- format: Joi.string().valid('html', 'markdown').default('html').description('Markup language for HTML ("html" or "markdown")'),
289
+ format: Joi.string()
290
+ .valid('html', 'markdown')
291
+ .default('html')
292
+ .description('Markup language for HTML ("html" or "markdown")')
293
+ .label('TemplateListFormat'),
281
294
  created: Joi.date().iso().example('2021-02-17T13:43:18.860Z').description('The time this template was created'),
282
295
  updated: Joi.date().iso().example('2021-02-17T13:43:18.860Z').description('The time this template was last updated')
283
296
  }).label('AccountTemplate')
@@ -345,7 +358,11 @@ async function init(args) {
345
358
  .example('Something about the template')
346
359
  .description('Optional description of the template')
347
360
  .label('TemplateDescription'),
348
- format: Joi.string().valid('html', 'markdown').default('html').description('Markup language for HTML ("html" or "markdown")'),
361
+ format: Joi.string()
362
+ .valid('html', 'markdown')
363
+ .default('html')
364
+ .description('Markup language for HTML ("html" or "markdown")')
365
+ .label('TemplateResponseFormat'),
349
366
  created: Joi.date().iso().example('2021-02-17T13:43:18.860Z').description('The time this template was created'),
350
367
  updated: Joi.date().iso().example('2021-02-17T13:43:18.860Z').description('The time this template was last updated'),
351
368
  content: Joi.object({
@@ -353,7 +370,11 @@ async function init(args) {
353
370
  text: templateSchemas.text,
354
371
  html: templateSchemas.html,
355
372
  previewText: templateSchemas.previewText,
356
- format: Joi.string().valid('html', 'markdown').default('html').description('Markup language for HTML ("html" or "markdown")')
373
+ format: Joi.string()
374
+ .valid('html', 'markdown')
375
+ .default('html')
376
+ .description('Markup language for HTML ("html" or "markdown")')
377
+ .label('TemplateContentFormat')
357
378
  }).label('RequestTemplateContent')
358
379
  }).label('AccountTemplateResponse'),
359
380
  failAction: 'log'
@@ -411,7 +432,7 @@ async function init(args) {
411
432
  deleted: Joi.boolean().truthy('Y', 'true', '1').falsy('N', 'false', 0).default(true).description('Was the template deleted'),
412
433
  account: accountIdSchema.required(),
413
434
  id: Joi.string().max(256).required().example('AAABgS-UcAYAAAABAA').description('Template ID')
414
- }).label('DeleteTemplateRequestResponse'),
435
+ }).label('DeleteTemplateResponse'),
415
436
  failAction: 'log'
416
437
  }
417
438
  }
@@ -480,7 +501,7 @@ async function init(args) {
480
501
  schema: Joi.object({
481
502
  deleted: Joi.boolean().truthy('Y', 'true', '1').falsy('N', 'false', 0).default(true).description('Were the templates flushed'),
482
503
  account: accountIdSchema.required()
483
- }).label('DeleteTemplateRequestResponse'),
504
+ }).label('FlushTemplatesResponse'),
484
505
  failAction: 'log'
485
506
  }
486
507
  }
package/lib/arf-detect.js CHANGED
@@ -27,7 +27,7 @@ const arfDetect = async messageInfo => {
27
27
 
28
28
  let returnPath;
29
29
 
30
- for (let attachment of messageInfo.attachments) {
30
+ for (let attachment of messageInfo.attachments || []) {
31
31
  switch (attachment.contentType.toLowerCase()) {
32
32
  case 'message/feedback-report': {
33
33
  // found a feedback report
package/lib/consts.js CHANGED
@@ -87,6 +87,22 @@ module.exports = {
87
87
  LIST_SUBSCRIBE_NOTIFY: 'listSubscribe',
88
88
  LIST_SUBSCRIBE_DESCRIPTION: 'Recipient Subscribed - A recipient re-subscribed to a list',
89
89
 
90
+ EXPORT_COMPLETED_NOTIFY: 'exportCompleted',
91
+ EXPORT_COMPLETED_DESCRIPTION: 'Export Completed - A bulk message export has completed successfully',
92
+
93
+ EXPORT_FAILED_NOTIFY: 'exportFailed',
94
+ EXPORT_FAILED_DESCRIPTION: 'Export Failed - A bulk message export has failed',
95
+
96
+ // Export defaults
97
+ DEFAULT_EXPORT_MAX_AGE: 24 * 60 * 60 * 1000, // 24 hours
98
+ DEFAULT_EXPORT_MAX_SIZE: 10 * 1024 * 1024 * 1024, // 10GB
99
+ DEFAULT_EXPORT_MAX_MESSAGES: 500000,
100
+ DEFAULT_EXPORT_MAX_CONCURRENT: 2,
101
+ DEFAULT_EXPORT_MAX_GLOBAL_CONCURRENT: 8,
102
+ DEFAULT_EXPORT_MAX_MESSAGE_SIZE: 50 * 1024 * 1024, // 50MB
103
+ DEFAULT_GMAIL_EXPORT_BATCH_SIZE: 10,
104
+ DEFAULT_OUTLOOK_EXPORT_BATCH_SIZE: 20,
105
+
90
106
  MAX_DAYS_STATS: 7,
91
107
 
92
108
  DEFAULT_MAX_LOG_LINES: 10000,
package/lib/db.js CHANGED
@@ -107,12 +107,14 @@ const reqisQueue = new Redis(REDIS_CONF);
107
107
 
108
108
  module.exports.queueConf = {
109
109
  connection: reqisQueue,
110
+ sharedConnection: true, // Prevent BullMQ from closing shared connection
110
111
  prefix: `${REDIS_PREFIX}bull`
111
112
  };
112
113
 
113
114
  const notifyQueue = new Queue('notify', module.exports.queueConf);
114
115
  const submitQueue = new Queue('submit', module.exports.queueConf);
115
116
  const documentsQueue = new Queue('documents', module.exports.queueConf);
117
+ const exportQueue = new Queue('export', module.exports.queueConf);
116
118
 
117
119
  const zExpungeScript = fs.readFileSync(pathlib.join(__dirname, '/lua/z-expunge.lua'), 'utf-8');
118
120
  const zSetScript = fs.readFileSync(pathlib.join(__dirname, 'lua/z-set.lua'), 'utf-8');
@@ -215,6 +217,7 @@ module.exports.redis = redis;
215
217
  module.exports.notifyQueue = notifyQueue;
216
218
  module.exports.submitQueue = submitQueue;
217
219
  module.exports.documentsQueue = documentsQueue;
220
+ module.exports.exportQueue = exportQueue;
218
221
 
219
222
  // do not set up the flow producer by default
220
223
  module.exports.getFlowProducer = () => new FlowProducer(module.exports.queueConf /*, Redis*/);
@@ -1984,7 +1984,7 @@ class BaseClient {
1984
1984
  // Store message in Redis
1985
1985
  await this.redis.hsetBuffer(`${REDIS_PREFIX}iaq:${this.account}`, queueId, msgEntry);
1986
1986
 
1987
- let queueKeep = (await settings.get('queueKeep')) || true;
1987
+ let queueKeep = (await settings.get('queueKeep')) ?? true;
1988
1988
 
1989
1989
  // Configure delivery retry settings
1990
1990
  let defaultDeliveryAttempts = await settings.get('deliveryAttempts');
@@ -2007,14 +2007,16 @@ class BaseClient {
2007
2007
 
2008
2008
  // Configure queue job options
2009
2009
  let queueName = 'queued';
2010
+ let retention = typeof queueKeep === 'number' ? { age: 24 * 3600, count: queueKeep } : queueKeep;
2010
2011
  let jobOpts = {
2011
2012
  jobId: queueId,
2012
- removeOnComplete: queueKeep,
2013
- removeOnFail: queueKeep,
2013
+ removeOnComplete: retention,
2014
+ removeOnFail: retention,
2014
2015
  attempts: typeof deliveryAttempts === 'number' ? deliveryAttempts : defaultDeliveryAttempts,
2015
2016
  backoff: {
2016
2017
  type: 'exponential',
2017
- delay: 5000
2018
+ delay: 5000,
2019
+ jitter: 0.2 // 20% randomization to prevent thundering herd
2018
2020
  }
2019
2021
  };
2020
2022
 
@@ -11,6 +11,7 @@ const he = require('he');
11
11
  const { BaseClient, metricsMeta } = require('./base-client');
12
12
  const { mimeHtml } = require('@postalsys/email-text-tools');
13
13
  const { emitChangeEvent } = require('../tools');
14
+ const crypto = require('crypto');
14
15
  const { Gateway } = require('../gateway');
15
16
 
16
17
  const {
@@ -20,11 +21,16 @@ const {
20
21
  EMAIL_SENT_NOTIFY,
21
22
  REDIS_PREFIX,
22
23
  AUTH_ERROR_NOTIFY,
23
- AUTH_SUCCESS_NOTIFY
24
+ AUTH_SUCCESS_NOTIFY,
25
+ DEFAULT_GMAIL_EXPORT_BATCH_SIZE
24
26
  } = require('../consts');
25
27
 
28
+ const settings = require('../settings');
29
+
26
30
  const { GMAIL_API_BASE, LIST_BATCH_SIZE, request: gmailApiRequest } = require('./gmail/gmail-api');
27
31
 
32
+ const MAX_GMAIL_BATCH_SIZE = 50;
33
+
28
34
  // Labels to exclude from folder listings
29
35
  const SKIP_LABELS = ['UNREAD', 'STARRED', 'IMPORTANT', 'CHAT', 'CATEGORY_PERSONAL'];
30
36
 
@@ -602,23 +608,39 @@ class GmailClient extends BaseClient {
602
608
 
603
609
  return folderData;
604
610
  })
605
- .filter(value => value)
606
- .sort((a, b) => {
607
- // Sort: INBOX first, then special folders, then alphabetical
608
- if (a.path === 'INBOX') {
609
- return -1;
610
- } else if (b.path === 'INBOX') {
611
- return 1;
612
- }
611
+ .filter(value => value);
612
+
613
+ // Add virtual "All Mail" folder for Gmail API
614
+ // This allows exporting all messages without scanning individual labels
615
+ mailboxes.unshift({
616
+ id: 'virtual_all',
617
+ path: '\\All',
618
+ delimiter: '/',
619
+ parentPath: '',
620
+ name: 'All Mail',
621
+ listed: true,
622
+ subscribed: false,
623
+ noSelect: true,
624
+ specialUse: '\\All',
625
+ specialUseSource: 'extension'
626
+ });
613
627
 
614
- if (a.specialUse && !b.specialUse) {
615
- return -1;
616
- } else if (!a.specialUse && b.specialUse) {
617
- return 1;
618
- }
628
+ mailboxes.sort((a, b) => {
629
+ // Sort: INBOX first, then special folders, then alphabetical
630
+ if (a.path === 'INBOX') {
631
+ return -1;
632
+ } else if (b.path === 'INBOX') {
633
+ return 1;
634
+ }
619
635
 
620
- return a.path.toLowerCase().localeCompare(b.path.toLowerCase());
621
- });
636
+ if (a.specialUse && !b.specialUse) {
637
+ return -1;
638
+ } else if (!a.specialUse && b.specialUse) {
639
+ return 1;
640
+ }
641
+
642
+ return a.path.toLowerCase().localeCompare(b.path.toLowerCase());
643
+ });
622
644
 
623
645
  return mailboxes;
624
646
  }
@@ -756,7 +778,7 @@ class GmailClient extends BaseClient {
756
778
  let nextPageCursor = pageCursor.nextPageCursor(listingResult.nextPageToken);
757
779
  let prevPageCursor = pageCursor.prevPageCursor();
758
780
 
759
- if (options.metadataOnly) {
781
+ if (options.metadataOnly || query.metadataOnly) {
760
782
  // Return just IDs without fetching full content
761
783
  return {
762
784
  total: messageCount,
@@ -769,6 +791,8 @@ class GmailClient extends BaseClient {
769
791
  }
770
792
 
771
793
  // Fetch message content for matching messages in batches
794
+ // Use format=minimal for minimalFields option (faster, returns only id, threadId, labelIds, internalDate, sizeEstimate)
795
+ const messageFormat = options.minimalFields ? 'minimal' : undefined;
772
796
 
773
797
  let promises = [];
774
798
 
@@ -782,14 +806,16 @@ class GmailClient extends BaseClient {
782
806
  throw entry.reason;
783
807
  }
784
808
  if (entry.value) {
785
- messageList.push(this.formatMessage(entry.value, { path }));
809
+ messageList.push(this.formatMessage(entry.value, { path, minimalFields: options.minimalFields }));
786
810
  }
787
811
  }
788
812
  promises = [];
789
813
  };
790
814
 
791
815
  for (let { id: message } of listingResult.messages || []) {
792
- promises.push(this.request(`${GMAIL_API_BASE}/gmail/v1/users/me/messages/${message}`));
816
+ let requestUrl = `${GMAIL_API_BASE}/gmail/v1/users/me/messages/${message}`;
817
+ let requestParams = messageFormat ? { format: messageFormat } : undefined;
818
+ promises.push(this.request(requestUrl, 'get', requestParams));
793
819
  if (promises.length > LIST_BATCH_SIZE) {
794
820
  await resolvePromises();
795
821
  }
@@ -1269,6 +1295,9 @@ class GmailClient extends BaseClient {
1269
1295
 
1270
1296
  let formattedMessage = this.formatMessage(messageData, { extended: true, textType: options.textType });
1271
1297
 
1298
+ // Resolve label IDs to human-readable names
1299
+ await this.resolveLabels(formattedMessage);
1300
+
1272
1301
  // Mark as seen if requested
1273
1302
  if (options.markAsSeen && (!formattedMessage.flags || !formattedMessage.flags.includes('\\Seen'))) {
1274
1303
  //
@@ -1325,6 +1354,64 @@ class GmailClient extends BaseClient {
1325
1354
  return formattedMessage;
1326
1355
  }
1327
1356
 
1357
+ /**
1358
+ * Fetches multiple messages in parallel for batch export operations
1359
+ * @param {string[]} emailIds - Array of message IDs
1360
+ * @param {Object} options - Fetch options
1361
+ * @returns {Object[]} Array of results with messageId, data, and error fields
1362
+ */
1363
+ async getMessages(emailIds, options) {
1364
+ options = options || {};
1365
+ await this.prepare();
1366
+
1367
+ // Pre-fetch labels to resolve label IDs to names
1368
+ const labelMap = new Map();
1369
+ try {
1370
+ const labelsResult = await this.getLabels();
1371
+ for (const label of labelsResult || []) {
1372
+ labelMap.set(label.id, label.name);
1373
+ }
1374
+ } catch (err) {
1375
+ this.logger.warn({ msg: 'Failed to fetch labels for export, using raw label IDs', account: this.account, err });
1376
+ }
1377
+
1378
+ const results = [];
1379
+ const settingsBatchSize = await settings.get('gmailExportBatchSize');
1380
+ const batchSize = Math.min(settingsBatchSize || DEFAULT_GMAIL_EXPORT_BATCH_SIZE, MAX_GMAIL_BATCH_SIZE);
1381
+
1382
+ for (let i = 0; i < emailIds.length; i += batchSize) {
1383
+ const batch = emailIds.slice(i, i + batchSize);
1384
+
1385
+ const batchResults = await Promise.all(
1386
+ batch.map(async emailId => {
1387
+ try {
1388
+ const requestQuery = { format: 'full' };
1389
+ const messageData = await this.request(`${GMAIL_API_BASE}/gmail/v1/users/me/messages/${emailId}`, 'get', requestQuery);
1390
+ const formattedMessage = this.formatMessage(messageData, { extended: true, textType: options.textType });
1391
+
1392
+ await this.resolveLabels(formattedMessage, labelMap);
1393
+
1394
+ return {
1395
+ messageId: emailId,
1396
+ data: formattedMessage,
1397
+ error: null
1398
+ };
1399
+ } catch (err) {
1400
+ return {
1401
+ messageId: emailId,
1402
+ data: null,
1403
+ error: { message: err.message, code: err.code, statusCode: err.statusCode }
1404
+ };
1405
+ }
1406
+ })
1407
+ );
1408
+
1409
+ results.push(...batchResults);
1410
+ }
1411
+
1412
+ return results;
1413
+ }
1414
+
1328
1415
  /**
1329
1416
  * Fetches text content for a message
1330
1417
  * @param {string} textId - Encoded text identifier
@@ -1521,19 +1608,45 @@ class GmailClient extends BaseClient {
1521
1608
  return false;
1522
1609
  }
1523
1610
 
1524
- // Prepare for Gmail API send
1525
- let contentType = 'message/rfc822';
1526
- let payload = raw;
1527
- let targetEndpoint = `/upload/gmail/v1/users/me/messages/send`;
1611
+ // Gmail JSON endpoint: 5MB body limit (~3.5MB raw before base64url overhead)
1612
+ // Gmail upload endpoint: 35MB raw RFC822
1613
+ let contentType;
1614
+ let payload;
1615
+ let targetEndpoint;
1616
+ const JSON_SEND_LIMIT = 3.5 * 1024 * 1024;
1528
1617
 
1529
- // Use different endpoint for thread replies
1530
- if (data?.reference?.threadId) {
1531
- targetEndpoint = `/gmail/v1/users/me/messages/send`;
1618
+ if (raw.length <= JSON_SEND_LIMIT) {
1619
+ // JSON endpoint with base64url encoding (retry-safe, no ArrayBuffer issues)
1532
1620
  contentType = 'application/json';
1533
- payload = {
1534
- raw: raw.toString('base64'),
1535
- threadId: data?.reference?.threadId
1536
- };
1621
+ payload = { raw: raw.toString('base64url') };
1622
+ targetEndpoint = `/gmail/v1/users/me/messages/send`;
1623
+ if (data?.reference?.threadId) {
1624
+ payload.threadId = data.reference.threadId;
1625
+ }
1626
+ } else if (data?.reference?.threadId) {
1627
+ // Large threaded reply: multipart upload preserves explicit threadId
1628
+ // via JSON metadata alongside the raw RFC822 message body
1629
+ const boundary = `ee_${crypto.randomBytes(16).toString('hex')}`;
1630
+ const metadata = JSON.stringify({ threadId: data.reference.threadId });
1631
+ const preamble = Buffer.from(
1632
+ `--${boundary}\r\n` +
1633
+ `Content-Type: application/json; charset=UTF-8\r\n` +
1634
+ `\r\n` +
1635
+ `${metadata}\r\n` +
1636
+ `--${boundary}\r\n` +
1637
+ `Content-Type: message/rfc822\r\n` +
1638
+ `\r\n`
1639
+ );
1640
+ const epilogue = Buffer.from(`\r\n--${boundary}--`);
1641
+
1642
+ contentType = `multipart/related; boundary=${boundary}`;
1643
+ payload = Buffer.concat([preamble, raw, epilogue]);
1644
+ targetEndpoint = `/upload/gmail/v1/users/me/messages/send?uploadType=multipart`;
1645
+ } else {
1646
+ // Large non-threaded message: simple upload with raw RFC822 Buffer
1647
+ contentType = 'message/rfc822';
1648
+ payload = raw;
1649
+ targetEndpoint = `/upload/gmail/v1/users/me/messages/send`;
1537
1650
  }
1538
1651
 
1539
1652
  // Send via Gmail API
@@ -1591,7 +1704,8 @@ class GmailClient extends BaseClient {
1591
1704
  await this.redis
1592
1705
  .multi()
1593
1706
  .hset(data.feedbackKey, 'success', 'true')
1594
- .expire(1 * 60 * 60);
1707
+ .expire(data.feedbackKey, 1 * 60 * 60)
1708
+ .exec();
1595
1709
  }
1596
1710
 
1597
1711
  return {
@@ -2042,6 +2156,35 @@ class GmailClient extends BaseClient {
2042
2156
  }
2043
2157
  }
2044
2158
 
2159
+ /**
2160
+ * Resolves label IDs to human-readable names on a formatted message.
2161
+ * Mutates formattedMessage.labels in place. Labels starting with '\\' are
2162
+ * treated as special-use labels and left as-is.
2163
+ *
2164
+ * @param {Object} formattedMessage - Message with a .labels array
2165
+ * @param {Map} [labelMap] - Optional pre-built id-to-name map (avoids an extra getLabels call)
2166
+ */
2167
+ async resolveLabels(formattedMessage, labelMap) {
2168
+ if (!Array.isArray(formattedMessage?.labels)) {
2169
+ return;
2170
+ }
2171
+
2172
+ if (!labelMap) {
2173
+ const labelsResult = await this.getLabels();
2174
+ labelMap = new Map();
2175
+ for (const label of labelsResult || []) {
2176
+ labelMap.set(label.id, label.name);
2177
+ }
2178
+ }
2179
+
2180
+ formattedMessage.labels = formattedMessage.labels.map(label => {
2181
+ if (label.startsWith('\\')) {
2182
+ return label;
2183
+ }
2184
+ return labelMap.get(label) || label;
2185
+ });
2186
+ }
2187
+
2045
2188
  /**
2046
2189
  * Resolves a label by path or ID
2047
2190
  * @param {string} path - Label path or ID
@@ -2292,7 +2435,7 @@ class GmailClient extends BaseClient {
2292
2435
  * @returns {Object} Formatted message
2293
2436
  */
2294
2437
  formatMessage(messageData, options) {
2295
- let { extended, path, textType } = options || {};
2438
+ let { extended, path, textType, minimalFields } = options || {};
2296
2439
 
2297
2440
  let date = messageData.internalDate && !isNaN(messageData.internalDate) ? new Date(Number(messageData.internalDate)) : undefined;
2298
2441
  if (date?.toString() === 'Invalid Date') {
@@ -2301,6 +2444,34 @@ class GmailClient extends BaseClient {
2301
2444
 
2302
2445
  let { flags, labels, category } = this.formatFlagsAndLabels(messageData);
2303
2446
 
2447
+ // For minimalFields mode (format=minimal), payload is not available
2448
+ // Return only basic fields: id, threadId, labelIds, internalDate, sizeEstimate
2449
+ if (minimalFields) {
2450
+ const result = {
2451
+ id: messageData.id,
2452
+ emailId: messageData.id || undefined,
2453
+ threadId: messageData.threadId || undefined,
2454
+ date: date ? date.toISOString() : undefined,
2455
+ flags,
2456
+ labels,
2457
+ category,
2458
+ unseen: !flags.includes('\\Seen') ? true : undefined,
2459
+ flagged: flags.includes('\\Flagged') ? true : undefined,
2460
+ draft: flags.includes('\\Draft') ? true : undefined,
2461
+ size: messageData.sizeEstimate
2462
+ };
2463
+
2464
+ // Set special-use based on labels
2465
+ for (let specialUseTag of ['\\Junk', '\\Sent', '\\Trash', '\\Inbox', '\\Drafts']) {
2466
+ if (result.labels && result.labels.includes(specialUseTag)) {
2467
+ result.messageSpecialUse = specialUseTag;
2468
+ break;
2469
+ }
2470
+ }
2471
+
2472
+ return result;
2473
+ }
2474
+
2304
2475
  let envelope = this.getEnvelope(messageData);
2305
2476
 
2306
2477
  // Extract all headers
@@ -2424,7 +2595,7 @@ class GmailClient extends BaseClient {
2424
2595
  * @returns {string} Formatted term
2425
2596
  */
2426
2597
  formatSearchTerm(term, quot = '"') {
2427
- if (typeof term === 'object' && term && Object.prototype.toString.apply(new Date()) === '[object Date]') {
2598
+ if (typeof term === 'object' && term && Object.prototype.toString.apply(term) === '[object Date]') {
2428
2599
  term = term.toISOString().substring(0, 10);
2429
2600
  }
2430
2601