emailengine-app 2.61.1 → 2.61.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (136) hide show
  1. package/CHANGELOG.md +17 -0
  2. package/data/google-crawlers.json +1 -1
  3. package/lib/account/account-state.js +248 -0
  4. package/lib/account.js +45 -193
  5. package/lib/api-routes/account-routes.js +1023 -0
  6. package/lib/api-routes/message-routes.js +1377 -0
  7. package/lib/consts.js +12 -2
  8. package/lib/email-client/base-client.js +282 -771
  9. package/lib/email-client/gmail/gmail-api.js +243 -0
  10. package/lib/email-client/gmail-client.js +145 -53
  11. package/lib/email-client/imap/mailbox.js +24 -698
  12. package/lib/email-client/imap/sync-operations.js +812 -0
  13. package/lib/email-client/imap-client.js +1 -1
  14. package/lib/email-client/message-builder.js +566 -0
  15. package/lib/email-client/notification-handler.js +314 -0
  16. package/lib/email-client/outlook/graph-api.js +326 -0
  17. package/lib/email-client/outlook-client.js +159 -113
  18. package/lib/email-client/smtp-pool-manager.js +196 -0
  19. package/lib/imapproxy/imap-server.js +3 -12
  20. package/lib/oauth/gmail.js +4 -4
  21. package/lib/oauth/mail-ru.js +30 -5
  22. package/lib/oauth/outlook.js +57 -3
  23. package/lib/oauth/pubsub/google.js +30 -11
  24. package/lib/oauth/scope-checker.js +202 -0
  25. package/lib/oauth2-apps.js +8 -4
  26. package/lib/redis-operations.js +484 -0
  27. package/lib/routes-ui.js +283 -2582
  28. package/lib/tools.js +4 -196
  29. package/lib/ui-routes/account-routes.js +1931 -0
  30. package/lib/ui-routes/admin-config-routes.js +1233 -0
  31. package/lib/ui-routes/admin-entities-routes.js +2367 -0
  32. package/lib/ui-routes/oauth-routes.js +992 -0
  33. package/lib/utils/network.js +237 -0
  34. package/package.json +10 -10
  35. package/sbom.json +1 -1
  36. package/static/js/app.js +5 -5
  37. package/static/licenses.html +79 -19
  38. package/translations/de.mo +0 -0
  39. package/translations/de.po +97 -86
  40. package/translations/en.mo +0 -0
  41. package/translations/en.po +80 -75
  42. package/translations/et.mo +0 -0
  43. package/translations/et.po +96 -86
  44. package/translations/fr.mo +0 -0
  45. package/translations/fr.po +97 -86
  46. package/translations/ja.mo +0 -0
  47. package/translations/ja.po +96 -86
  48. package/translations/messages.pot +105 -91
  49. package/translations/nl.mo +0 -0
  50. package/translations/nl.po +98 -86
  51. package/translations/pl.mo +0 -0
  52. package/translations/pl.po +96 -86
  53. package/views/account/security.hbs +4 -4
  54. package/views/accounts/account.hbs +13 -13
  55. package/views/accounts/register/imap-server.hbs +12 -12
  56. package/views/config/document-store/pre-processing/index.hbs +4 -2
  57. package/views/config/oauth/app.hbs +6 -7
  58. package/views/config/oauth/index.hbs +2 -2
  59. package/views/config/service.hbs +3 -4
  60. package/views/dashboard.hbs +5 -7
  61. package/views/error.hbs +22 -7
  62. package/views/gateways/gateway.hbs +2 -2
  63. package/views/partials/add_account_modal.hbs +7 -10
  64. package/views/partials/document_store_header.hbs +1 -1
  65. package/views/partials/editor_scope_info.hbs +0 -1
  66. package/views/partials/oauth_config_header.hbs +1 -1
  67. package/views/partials/side_menu.hbs +3 -3
  68. package/views/partials/webhook_form.hbs +2 -2
  69. package/views/templates/index.hbs +1 -1
  70. package/views/templates/template.hbs +8 -8
  71. package/views/tokens/index.hbs +6 -6
  72. package/views/tokens/new.hbs +1 -1
  73. package/views/webhooks/index.hbs +4 -4
  74. package/views/webhooks/webhook.hbs +7 -7
  75. package/workers/api.js +148 -2436
  76. package/workers/smtp.js +2 -1
  77. package/lib/imapproxy/imap-core/test/client.js +0 -46
  78. package/lib/imapproxy/imap-core/test/fixtures/append.eml +0 -1196
  79. package/lib/imapproxy/imap-core/test/fixtures/chunks.js +0 -44
  80. package/lib/imapproxy/imap-core/test/fixtures/fix1.eml +0 -6
  81. package/lib/imapproxy/imap-core/test/fixtures/fix2.eml +0 -599
  82. package/lib/imapproxy/imap-core/test/fixtures/fix3.eml +0 -32
  83. package/lib/imapproxy/imap-core/test/fixtures/fix4.eml +0 -6
  84. package/lib/imapproxy/imap-core/test/fixtures/mimetorture.eml +0 -599
  85. package/lib/imapproxy/imap-core/test/fixtures/mimetorture.js +0 -2740
  86. package/lib/imapproxy/imap-core/test/fixtures/mimetorture.json +0 -1411
  87. package/lib/imapproxy/imap-core/test/fixtures/mimetree.js +0 -85
  88. package/lib/imapproxy/imap-core/test/fixtures/nodemailer.eml +0 -582
  89. package/lib/imapproxy/imap-core/test/fixtures/ryan_finnie_mime_torture.eml +0 -599
  90. package/lib/imapproxy/imap-core/test/fixtures/simple.eml +0 -42
  91. package/lib/imapproxy/imap-core/test/fixtures/simple.json +0 -164
  92. package/lib/imapproxy/imap-core/test/imap-compile-stream-test.js +0 -671
  93. package/lib/imapproxy/imap-core/test/imap-compiler-test.js +0 -272
  94. package/lib/imapproxy/imap-core/test/imap-indexer-test.js +0 -236
  95. package/lib/imapproxy/imap-core/test/imap-parser-test.js +0 -922
  96. package/lib/imapproxy/imap-core/test/memory-notifier.js +0 -129
  97. package/lib/imapproxy/imap-core/test/prepare.sh +0 -74
  98. package/lib/imapproxy/imap-core/test/protocol-test.js +0 -1756
  99. package/lib/imapproxy/imap-core/test/search-test.js +0 -1356
  100. package/lib/imapproxy/imap-core/test/test-client.js +0 -152
  101. package/lib/imapproxy/imap-core/test/test-server.js +0 -623
  102. package/lib/imapproxy/imap-core/test/tools-test.js +0 -22
  103. package/test/api-test.js +0 -899
  104. package/test/autoreply-test.js +0 -327
  105. package/test/bounce-test.js +0 -151
  106. package/test/complaint-test.js +0 -256
  107. package/test/fixtures/autoreply/LICENSE +0 -27
  108. package/test/fixtures/autoreply/rfc3834-01.eml +0 -23
  109. package/test/fixtures/autoreply/rfc3834-02.eml +0 -24
  110. package/test/fixtures/autoreply/rfc3834-03.eml +0 -26
  111. package/test/fixtures/autoreply/rfc3834-04.eml +0 -48
  112. package/test/fixtures/autoreply/rfc3834-05.eml +0 -19
  113. package/test/fixtures/autoreply/rfc3834-06.eml +0 -59
  114. package/test/fixtures/bounces/163.eml +0 -2521
  115. package/test/fixtures/bounces/fastmail.eml +0 -242
  116. package/test/fixtures/bounces/gmail.eml +0 -252
  117. package/test/fixtures/bounces/hotmail.eml +0 -655
  118. package/test/fixtures/bounces/mailru.eml +0 -121
  119. package/test/fixtures/bounces/outlook.eml +0 -1107
  120. package/test/fixtures/bounces/postfix.eml +0 -101
  121. package/test/fixtures/bounces/rambler.eml +0 -116
  122. package/test/fixtures/bounces/workmail.eml +0 -142
  123. package/test/fixtures/bounces/yahoo.eml +0 -139
  124. package/test/fixtures/bounces/zoho.eml +0 -83
  125. package/test/fixtures/bounces/zonemta.eml +0 -100
  126. package/test/fixtures/complaints/LICENSE +0 -27
  127. package/test/fixtures/complaints/amazonses.eml +0 -72
  128. package/test/fixtures/complaints/dmarc.eml +0 -59
  129. package/test/fixtures/complaints/hotmail.eml +0 -49
  130. package/test/fixtures/complaints/optout.eml +0 -40
  131. package/test/fixtures/complaints/standard-arf.eml +0 -68
  132. package/test/fixtures/complaints/yahoo.eml +0 -68
  133. package/test/oauth2-apps-test.js +0 -301
  134. package/test/sendonly-test.js +0 -160
  135. package/test/test-config.js +0 -34
  136. package/test/webhooks-server.js +0 -39
@@ -1,22 +1,11 @@
1
1
  'use strict';
2
2
 
3
3
  const crypto = require('crypto');
4
- const {
5
- serialize,
6
- unserialize,
7
- compareExisting,
8
- normalizePath,
9
- download,
10
- filterEmptyObjectValues,
11
- validUidValidity,
12
- calculateFetchBackoff,
13
- readEnvValue
14
- } = require('../../tools');
4
+ const { serialize, unserialize, compareExisting, normalizePath, download, filterEmptyObjectValues, validUidValidity } = require('../../tools');
15
5
  const msgpack = require('msgpack5')();
16
6
  const he = require('he');
17
7
  const libmime = require('libmime');
18
8
  const settings = require('../../settings');
19
- const config = require('@zone-eu/wild-config');
20
9
  const { bounceDetect } = require('../../bounce-detect');
21
10
  const { arfDetect } = require('../../arf-detect');
22
11
  const appendList = require('../../append-list');
@@ -41,41 +30,18 @@ const {
41
30
  REDIS_PREFIX,
42
31
  MAX_INLINE_ATTACHMENT_SIZE,
43
32
  MAX_ALLOWED_DOWNLOAD_SIZE,
44
- DEFAULT_FETCH_BATCH_SIZE,
45
33
  MAILBOX_HASH
46
34
  } = require('../../consts');
47
35
 
48
- // Configurable batch size for fetching messages (default: 250)
49
- const FETCH_BATCH_SIZE = Number(readEnvValue('EENGINE_FETCH_BATCH_SIZE') || config.service.fetchBatchSize) || DEFAULT_FETCH_BATCH_SIZE;
50
-
51
- // Do not check for flag updates using full sync more often than this value (30 minutes)
52
- const FULL_SYNC_DELAY = 30 * 60 * 1000;
53
-
54
- /**
55
- * Calculates the next range of sequence numbers to fetch based on the last fetched range
56
- * @param {Number} totalMessages - Total number of messages in the mailbox
57
- * @param {String} lastRange - Last fetched range in format "start:end" or "start:*"
58
- * @returns {String|false} Next range to fetch or false if no more messages
59
- */
60
- function getFetchRange(totalMessages, lastRange) {
61
- let lastEndMarker = lastRange ? lastRange.split(':').pop() : false;
62
- if (lastEndMarker === '*') {
63
- // Already fetched to the end
64
- return false;
65
- }
66
- let lastSeq = lastRange ? Number(lastEndMarker) : 0;
67
- let startSeq = lastSeq + 1;
68
- if (startSeq > totalMessages) {
69
- // No more messages to fetch
70
- return false;
71
- }
72
- let endMarker = startSeq + FETCH_BATCH_SIZE - 1;
73
- if (endMarker >= totalMessages) {
74
- // Use * to fetch to the end
75
- endMarker = '*';
76
- }
77
- return `${startSeq}:${endMarker}`;
78
- }
36
+ const {
37
+ SyncOperations,
38
+ hasUidValidityChanged,
39
+ hasNoModseqChanges,
40
+ canUseCondstorePartialSync,
41
+ canUseSimplePartialSync,
42
+ canSkipSync,
43
+ FULL_SYNC_DELAY
44
+ } = require('./sync-operations');
79
45
 
80
46
  /**
81
47
  * Represents a single IMAP mailbox/folder and handles all operations on it
@@ -115,6 +81,9 @@ class Mailbox {
115
81
 
116
82
  this.synced = false; // Whether initial sync is complete
117
83
  this.syncing = false; // Whether currently syncing
84
+
85
+ // Sync operations handler
86
+ this.syncOps = new SyncOperations(this);
118
87
  }
119
88
 
120
89
  /**
@@ -1828,646 +1797,19 @@ class Mailbox {
1828
1797
 
1829
1798
  /**
1830
1799
  * Performs full synchronization based on indexer type
1800
+ * Delegates to SyncOperations
1831
1801
  */
1832
1802
  async fullSync() {
1833
- const imapIndexer = this.imapIndexer;
1834
-
1835
- this.logger.trace({ msg: 'Running full sync', imapIndexer });
1836
-
1837
- switch (imapIndexer) {
1838
- case 'fast':
1839
- return this.runFastSync();
1840
- case 'full':
1841
- default:
1842
- return this.runFullSync();
1843
- }
1803
+ return this.syncOps.fullSync();
1844
1804
  }
1845
1805
 
1846
1806
  /**
1847
1807
  * Performs partial synchronization based on indexer type
1808
+ * Delegates to SyncOperations
1848
1809
  * @param {Object} storedStatus - Current stored mailbox status
1849
1810
  */
1850
1811
  async partialSync(storedStatus) {
1851
- const imapIndexer = this.imapIndexer;
1852
-
1853
- this.logger.trace({ msg: 'Running partial sync', imapIndexer });
1854
-
1855
- switch (imapIndexer) {
1856
- case 'fast':
1857
- return this.runFastSync(storedStatus);
1858
- case 'full':
1859
- default:
1860
- return this.runPartialSync(storedStatus);
1861
- }
1862
- }
1863
-
1864
- /**
1865
- * Fast sync mode - only tracks new messages, doesn't maintain full message list
1866
- * More efficient for large mailboxes where we only care about new messages
1867
- * @param {Object} storedStatus - Current stored mailbox status
1868
- */
1869
- async runFastSync(storedStatus) {
1870
- storedStatus = storedStatus || (await this.getStoredStatus());
1871
- let mailboxStatus = this.getMailboxStatus();
1872
-
1873
- let lock = await this.getMailboxLock(null, { description: 'Fast sync' });
1874
- this.connection.syncing = true;
1875
- this.syncing = true;
1876
- try {
1877
- if (!this.connection.imapClient) {
1878
- throw new Error('IMAP connection not available');
1879
- }
1880
-
1881
- let knownUidNext = typeof storedStatus.uidNext === 'number' ? storedStatus.uidNext || 1 : 1;
1882
-
1883
- if (knownUidNext && mailboxStatus.messages) {
1884
- // detected new emails
1885
- let fields = { uid: true, flags: true, modseq: true, emailId: true, labels: true, internalDate: true };
1886
-
1887
- let imapClient = this.connection.imapClient;
1888
-
1889
- // If we have not yet scanned this folder, then start by finding the earliest matching email
1890
- if (typeof storedStatus.uidNext !== 'number' && this.connection.notifyFrom && this.connection.notifyFrom < new Date()) {
1891
- // Find first message after notifyFrom date
1892
- let matchingMessages = await imapClient.search({ since: this.connection.notifyFrom }, { uid: true });
1893
- if (matchingMessages) {
1894
- let earliestUid = matchingMessages[0];
1895
- if (earliestUid) {
1896
- knownUidNext = earliestUid;
1897
- } else if (mailboxStatus.uidNext) {
1898
- // no match, start from newest
1899
- knownUidNext = mailboxStatus.uidNext;
1900
- }
1901
- }
1902
- }
1903
-
1904
- let range = `${knownUidNext}:*`;
1905
- let opts = {
1906
- uid: true
1907
- };
1908
-
1909
- // Fetch messages with retry logic
1910
- let fetchCompleted = false;
1911
- let fetchRetryCount = 0;
1912
-
1913
- while (!fetchCompleted) {
1914
- try {
1915
- let messages = [];
1916
-
1917
- // Fetch all messages in range
1918
- for await (let messageData of imapClient.fetch(range, fields, opts)) {
1919
- if (!messageData || !messageData.uid) {
1920
- //TODO: support partial responses
1921
- this.logger.debug({ msg: 'Partial FETCH response', code: 'partial_fetch', query: { range, fields, opts } });
1922
- continue;
1923
- }
1924
-
1925
- // ignore Recent flag
1926
- messageData.flags.delete('\\Recent');
1927
-
1928
- messages.push(messageData);
1929
- }
1930
- // ensure that messages are sorted by UID
1931
- messages = messages.sort((a, b) => a.uid - b.uid);
1932
-
1933
- // Process each new message
1934
- for (let messageData of messages) {
1935
- // Update uidNext if this is a new message
1936
- let updated = await this.connection.redis.hUpdateBigger(this.getMailboxKey(), 'uidNext', messageData.uid + 1, messageData.uid + 1);
1937
-
1938
- if (updated) {
1939
- // new email! Queue for processing
1940
- await this.connection.redis.zadd(
1941
- this.getNotificationsKey(),
1942
- messageData.uid,
1943
- JSON.stringify({
1944
- uid: messageData.uid,
1945
- flags: messageData.flags,
1946
- internalDate:
1947
- (messageData.internalDate &&
1948
- typeof messageData.internalDate.toISOString === 'function' &&
1949
- messageData.internalDate.toISOString()) ||
1950
- null
1951
- })
1952
- );
1953
- }
1954
- }
1955
-
1956
- try {
1957
- // clear failure flag
1958
- await this.connection.redis.hdel(this.connection.getAccountKey(), 'syncError');
1959
- } catch (err) {
1960
- // ignore
1961
- }
1962
- fetchCompleted = true;
1963
- } catch (err) {
1964
- try {
1965
- // set failure flag
1966
- await this.connection.redis.hSetExists(
1967
- this.connection.getAccountKey(),
1968
- 'syncError',
1969
- JSON.stringify({
1970
- path: this.path,
1971
- time: new Date().toISOString(),
1972
- error: {
1973
- error: err.message,
1974
- responseStatus: err.responseStatus,
1975
- responseText: err.responseText
1976
- }
1977
- })
1978
- );
1979
- } catch (err) {
1980
- // ignore
1981
- }
1982
-
1983
- // Retry with exponential backoff
1984
- if (!imapClient.usable) {
1985
- // nothing to do here, connection closed
1986
- this.logger.error({ msg: `FETCH failed, connection already closed, not retrying`, err });
1987
- return;
1988
- }
1989
-
1990
- const fetchRetryDelay = calculateFetchBackoff(++fetchRetryCount);
1991
- this.logger.error({ msg: `FETCH failed, retrying in ${Math.round(fetchRetryDelay / 1000)}s`, err });
1992
- await new Promise(r => setTimeout(r, fetchRetryDelay));
1993
-
1994
- if (!imapClient.usable) {
1995
- // nothing to do here, connection closed
1996
- this.logger.error({ msg: `FETCH failed, connection already closed, not retrying`, err });
1997
- return;
1998
- }
1999
- }
2000
- }
2001
- }
2002
-
2003
- await this.updateStoredStatus(this.getMailboxStatus());
2004
-
2005
- await this.publishSyncedEvents(storedStatus);
2006
- } finally {
2007
- lock.release();
2008
- this.connection.syncing = false;
2009
- this.syncing = false;
2010
- }
2011
- }
2012
-
2013
- /**
2014
- * Partial sync - fetches only changed messages using MODSEQ or UID range
2015
- * Used for incremental updates when we know something changed
2016
- * @param {Object} storedStatus - Current stored mailbox status
2017
- */
2018
- async runPartialSync(storedStatus) {
2019
- storedStatus = storedStatus || (await this.getStoredStatus());
2020
- let mailboxStatus = this.getMailboxStatus();
2021
-
2022
- let lock = await this.getMailboxLock(null, { description: 'Partial sync' });
2023
- this.connection.syncing = true;
2024
- this.syncing = true;
2025
- try {
2026
- if (!this.connection.imapClient) {
2027
- throw new Error('IMAP connection not available');
2028
- }
2029
-
2030
- let fields = { uid: true, flags: true, modseq: true, emailId: true, labels: true, internalDate: true };
2031
- let range = '1:*';
2032
- let opts = {
2033
- uid: true
2034
- };
2035
-
2036
- // Use CONDSTORE if available for efficient change detection
2037
- if (this.connection.imapClient.enabled.has('CONDSTORE') && storedStatus.highestModseq) {
2038
- // Only fetch messages changed since last known MODSEQ
2039
- opts.changedSince = storedStatus.highestModseq;
2040
- } else if (storedStatus.uidNext) {
2041
- // Fall back to fetching new messages only
2042
- range = `${storedStatus.uidNext}:*`;
2043
- }
2044
-
2045
- if (mailboxStatus.messages) {
2046
- // only fetch messages if there are some
2047
- let fetchCompleted = false;
2048
- let fetchRetryCount = 0;
2049
- while (!fetchCompleted) {
2050
- // Get fresh imapClient reference inside retry loop
2051
- let imapClient = this.connection.imapClient;
2052
- if (!imapClient || !imapClient.usable) {
2053
- this.logger.error({ msg: 'IMAP client not available for partial sync' });
2054
- throw new Error('IMAP connection not available');
2055
- }
2056
-
2057
- try {
2058
- // Fetch and process each message
2059
- for await (let messageData of imapClient.fetch(range, fields, opts)) {
2060
- if (!messageData || !messageData.uid) {
2061
- //TODO: support partial responses
2062
- this.logger.debug({ msg: 'Partial FETCH response', code: 'partial_fetch', query: { range, fields, opts } });
2063
- continue;
2064
- }
2065
-
2066
- // ignore Recent flag
2067
- messageData.flags.delete('\\Recent');
2068
-
2069
- let storedMessage = await this.entryListGet(messageData.uid, { uid: true });
2070
-
2071
- let changes;
2072
- if (!storedMessage) {
2073
- // New message
2074
- let seq = await this.entryListSet(messageData);
2075
- if (seq) {
2076
- // Queue for processing
2077
- await this.connection.redis.zadd(
2078
- this.getNotificationsKey(),
2079
- messageData.uid,
2080
- JSON.stringify({
2081
- uid: messageData.uid,
2082
- flags: messageData.flags,
2083
- internalDate:
2084
- (messageData.internalDate &&
2085
- typeof messageData.internalDate.toISOString === 'function' &&
2086
- messageData.internalDate.toISOString()) ||
2087
- null
2088
- })
2089
- );
2090
- }
2091
- } else if ((changes = compareExisting(storedMessage.entry, messageData))) {
2092
- // Existing message with changes
2093
- let seq = await this.entryListSet(messageData);
2094
- if (seq) {
2095
- await this.processChanges(messageData, changes);
2096
- }
2097
- }
2098
- }
2099
- try {
2100
- // clear failure flag
2101
- await this.connection.redis.hdel(this.connection.getAccountKey(), 'syncError');
2102
- } catch (err) {
2103
- // ignore
2104
- }
2105
- fetchCompleted = true;
2106
- } catch (err) {
2107
- try {
2108
- // set failure flag
2109
- await this.connection.redis.hSetExists(
2110
- this.connection.getAccountKey(),
2111
- 'syncError',
2112
- JSON.stringify({
2113
- path: this.path,
2114
- time: new Date().toISOString(),
2115
- error: {
2116
- error: err.message,
2117
- responseStatus: err.responseStatus,
2118
- responseText: err.responseText
2119
- }
2120
- })
2121
- );
2122
- } catch (err) {
2123
- // ignore
2124
- }
2125
-
2126
- // Retry with exponential backoff
2127
- if (!imapClient.usable) {
2128
- // nothing to do here, connection closed
2129
- this.logger.error({ msg: `FETCH failed, connection already closed, not retrying` });
2130
- return;
2131
- }
2132
-
2133
- const fetchRetryDelay = calculateFetchBackoff(++fetchRetryCount);
2134
- this.logger.error({ msg: `FETCH failed, retrying in ${Math.round(fetchRetryDelay / 1000)}s` });
2135
- await new Promise(r => setTimeout(r, fetchRetryDelay));
2136
-
2137
- if (!imapClient.usable) {
2138
- // nothing to do here, connection closed
2139
- this.logger.error({ msg: `FETCH failed, connection already closed, not retrying` });
2140
- return;
2141
- }
2142
- }
2143
- }
2144
- }
2145
-
2146
- await this.updateStoredStatus(this.getMailboxStatus());
2147
-
2148
- await this.publishSyncedEvents(storedStatus);
2149
- } finally {
2150
- lock.release();
2151
- this.connection.syncing = false;
2152
- this.syncing = false;
2153
- }
2154
- }
2155
-
2156
- /**
2157
- * Full sync - fetches all messages and detects additions, deletions, and changes
2158
- * Most thorough but slowest sync method
2159
- */
2160
- async runFullSync() {
2161
- let fields = { uid: true, flags: true, modseq: true, emailId: true, labels: true, internalDate: true };
2162
- let opts = {};
2163
-
2164
- let lock = await this.getMailboxLock(null, { description: 'Full sync' });
2165
- this.connection.syncing = true;
2166
- this.syncing = true;
2167
- try {
2168
- // Generate unique ID for this sync loop to track batch ordering
2169
- const loopId = crypto.randomUUID();
2170
-
2171
- // Wait for next tick to ensure ImapFlow has processed all untagged responses from SELECT
2172
- await new Promise(resolve => setImmediate(resolve));
2173
-
2174
- let mailboxStatus = this.getMailboxStatus();
2175
-
2176
- this.logger.debug({
2177
- msg: 'Starting full sync',
2178
- code: 'full_sync_start',
2179
- loopId,
2180
- mailboxStatus,
2181
- imapClientExists: mailboxStatus.messages
2182
- });
2183
-
2184
- // Track highest sequence number seen
2185
- let seqMax = 0;
2186
- let changes;
2187
-
2188
- // Get current message count for deletion detection
2189
- let storedMaxSeqOld = await this.connection.redis.zcard(this.getMessagesKey());
2190
-
2191
- let responseCounters = {
2192
- empty: 0,
2193
- partial: 0,
2194
- messages: 0
2195
- };
2196
-
2197
- if (mailboxStatus.messages) {
2198
- this.logger.debug({
2199
- msg: 'Running FETCH',
2200
- code: 'run_fetch',
2201
- query: { fields, opts },
2202
- expectedMessages: mailboxStatus.messages,
2203
- mailbox: mailboxStatus,
2204
- maxBatchSize: FETCH_BATCH_SIZE,
2205
- expectedBatches: Math.ceil(mailboxStatus.messages / FETCH_BATCH_SIZE)
2206
- });
2207
-
2208
- // Process messages in batches to avoid memory issues
2209
- let range = false;
2210
- let lastHighestUid = 0;
2211
- let batchNumber = 0;
2212
- // process messages in batches
2213
- while ((range = getFetchRange(mailboxStatus.messages, range))) {
2214
- batchNumber++;
2215
- this.logger.debug({
2216
- msg: 'Processing batch',
2217
- code: 'fetch_batch',
2218
- loopId,
2219
- batchNumber,
2220
- range,
2221
- totalMessages: mailboxStatus.messages,
2222
- previousRange: batchNumber > 1 ? 'calculated' : 'initial'
2223
- });
2224
- let fetchCompleted = false;
2225
- let fetchRetryCount = 0;
2226
- while (!fetchCompleted) {
2227
- // Get fresh imapClient reference inside retry loop
2228
- // This ensures we use the current connection state
2229
- const imapClient = this.connection.imapClient;
2230
- if (!imapClient || !imapClient.usable) {
2231
- this.logger.error({ msg: 'IMAP client not available for FETCH' });
2232
- throw new Error('IMAP connection not available');
2233
- }
2234
-
2235
- try {
2236
- this.logger.debug({
2237
- msg: 'Starting FETCH command',
2238
- code: 'fetch_start',
2239
- loopId,
2240
- batchNumber,
2241
- range,
2242
- retryCount: fetchRetryCount,
2243
- totalMessages: mailboxStatus.messages
2244
- });
2245
-
2246
- for await (let messageData of imapClient.fetch(range, fields, opts)) {
2247
- if (!messageData) {
2248
- this.logger.debug({ msg: 'Empty FETCH response', code: 'empty_fetch', query: { range, fields, opts } });
2249
- responseCounters.empty++;
2250
- continue;
2251
- }
2252
-
2253
- if (!messageData.uid || (fields.flags && !messageData.flags)) {
2254
- // TODO: support partial responses
2255
- // For now, without UID or FLAGS there's nothing to do
2256
- this.logger.debug({
2257
- msg: 'Partial FETCH response',
2258
- code: 'partial_fetch',
2259
- query: { range, fields, opts },
2260
- responseKeys: Object.keys(messageData)
2261
- });
2262
- responseCounters.partial++;
2263
- continue;
2264
- }
2265
-
2266
- if (messageData.uid <= lastHighestUid) {
2267
- // already processed in the previous batch
2268
- // probably an older email was deleted which shifted message entries
2269
- continue;
2270
- }
2271
- lastHighestUid = messageData.uid;
2272
-
2273
- responseCounters.messages++;
2274
-
2275
- if (fields.internalDate && !messageData.internalDate) {
2276
- this.logger.debug({
2277
- msg: 'Missing INTERNALDATE',
2278
- code: 'fetch_date_missing',
2279
- query: { range, fields, opts },
2280
- responseKeys: Object.keys(messageData)
2281
- });
2282
- }
2283
-
2284
- // ignore Recent flag
2285
- messageData.flags.delete('\\Recent');
2286
-
2287
- if (messageData.seq > seqMax) {
2288
- seqMax = messageData.seq;
2289
- }
2290
-
2291
- let storedMessage = await this.entryListGet(messageData.uid, { uid: true });
2292
- if (!storedMessage) {
2293
- // New message
2294
- let seq = await this.entryListSet(messageData);
2295
- if (seq) {
2296
- await this.connection.redis.zadd(
2297
- this.getNotificationsKey(),
2298
- messageData.uid,
2299
- JSON.stringify({
2300
- uid: messageData.uid,
2301
- flags: messageData.flags,
2302
- internalDate:
2303
- (messageData.internalDate &&
2304
- typeof messageData.internalDate.toISOString === 'function' &&
2305
- messageData.internalDate.toISOString()) ||
2306
- null
2307
- })
2308
- );
2309
- }
2310
- } else {
2311
- // Check for deleted messages between stored and current sequence
2312
- let diff = storedMessage.seq - messageData.seq;
2313
- if (diff) {
2314
- this.logger.trace({ msg: 'Deleted range', inloop: true, diff, start: messageData.seq });
2315
- }
2316
- // Process deletions
2317
- for (let i = diff - 1; i >= 0; i--) {
2318
- let seq = messageData.seq + i;
2319
- let deletedEntry = await this.entryListExpunge(seq);
2320
- if (deletedEntry) {
2321
- await this.processDeleted(deletedEntry);
2322
- }
2323
- }
2324
-
2325
- // Check for changes
2326
- if ((changes = compareExisting(storedMessage.entry, messageData))) {
2327
- let seq = await this.entryListSet(messageData);
2328
- if (seq) {
2329
- await this.processChanges(messageData, changes);
2330
- }
2331
- }
2332
- }
2333
- }
2334
-
2335
- try {
2336
- // clear failure flag
2337
- await this.connection.redis.hdel(this.connection.getAccountKey(), 'syncError');
2338
- } catch (err) {
2339
- // ignore
2340
- }
2341
-
2342
- this.logger.debug({
2343
- msg: 'FETCH completed successfully',
2344
- code: 'fetch_success',
2345
- loopId,
2346
- batchNumber,
2347
- range,
2348
- retryCount: fetchRetryCount
2349
- });
2350
-
2351
- fetchCompleted = true;
2352
- } catch (err) {
2353
- this.logger.error({
2354
- msg: 'FETCH failed',
2355
- code: 'fetch_error',
2356
- loopId,
2357
- batchNumber,
2358
- range,
2359
- retryCount: fetchRetryCount,
2360
- totalMessages: mailboxStatus.messages,
2361
- error: err.message,
2362
- responseStatus: err.responseStatus,
2363
- responseText: err.responseText
2364
- });
2365
-
2366
- if (!imapClient.usable) {
2367
- // nothing to do here, connection closed
2368
- this.logger.error({ msg: `FETCH failed, connection already closed, not retrying`, loopId });
2369
- return;
2370
- }
2371
-
2372
- try {
2373
- // set failure flag
2374
- await this.connection.redis.hSetExists(
2375
- this.connection.getAccountKey(),
2376
- 'syncError',
2377
- JSON.stringify({
2378
- path: this.path,
2379
- time: new Date().toISOString(),
2380
- error: {
2381
- error: err.message,
2382
- responseStatus: err.responseStatus,
2383
- responseText: err.responseText
2384
- }
2385
- })
2386
- );
2387
- } catch (err) {
2388
- // ignore
2389
- }
2390
-
2391
- // Retry with exponential backoff
2392
- const fetchRetryDelay = calculateFetchBackoff(++fetchRetryCount);
2393
- this.logger.error({ msg: `FETCH failed, retrying in ${Math.round(fetchRetryDelay / 1000)}s`, loopId, batchNumber });
2394
- await new Promise(r => setTimeout(r, fetchRetryDelay));
2395
-
2396
- if (!imapClient.usable) {
2397
- // nothing to do here, connection closed
2398
- this.logger.error({ msg: `FETCH failed, connection already closed, not retrying`, loopId });
2399
- return;
2400
- }
2401
-
2402
- // Verify we're still on the correct mailbox after the delay
2403
- // Another operation might have changed the mailbox while we were waiting
2404
- const currentMailbox = this.connection.imapClient.mailbox;
2405
- if (!currentMailbox || currentMailbox.path !== this.path) {
2406
- this.logger.error({
2407
- msg: 'Mailbox changed during retry delay, aborting sync',
2408
- expectedPath: this.path,
2409
- currentPath: currentMailbox ? currentMailbox.path : 'none',
2410
- loopId
2411
- });
2412
- throw new Error('Mailbox changed during sync operation');
2413
- }
2414
-
2415
- // Refresh mailbox status in case it changed
2416
- const oldMailboxMessages = mailboxStatus.messages;
2417
- mailboxStatus = this.getMailboxStatus();
2418
-
2419
- this.logger.debug({
2420
- msg: 'Refreshed mailbox status after error',
2421
- code: 'mailbox_status_refresh',
2422
- loopId,
2423
- batchNumber,
2424
- oldMessages: oldMailboxMessages,
2425
- newMessages: mailboxStatus.messages,
2426
- range
2427
- });
2428
- }
2429
- }
2430
- }
2431
- }
2432
-
2433
- // Delete any messages that weren't seen in this sync
2434
- let storedMaxSeq = await this.connection.redis.zcard(this.getMessagesKey());
2435
- let diff = storedMaxSeq - seqMax;
2436
- if (diff) {
2437
- this.logger.trace({
2438
- msg: 'Deleted range',
2439
- inloop: false,
2440
- diff,
2441
- start: seqMax + 1,
2442
- messagesKey: this.getMessagesKey(),
2443
- zcard: storedMaxSeq,
2444
- zcardOld: storedMaxSeqOld,
2445
- responseCounters
2446
- });
2447
- }
2448
-
2449
- // Process remaining deletions
2450
- for (let i = diff - 1; i >= 0; i--) {
2451
- let seq = seqMax + i + 1;
2452
- let deletedEntry = await this.entryListExpunge(seq);
2453
- if (deletedEntry) {
2454
- await this.processDeleted(deletedEntry);
2455
- }
2456
- }
2457
-
2458
- // Update status with full sync timestamp
2459
- let status = this.getMailboxStatus();
2460
- status.lastFullSync = new Date();
2461
-
2462
- await this.updateStoredStatus(status);
2463
- let storedStatus = await this.getStoredStatus();
2464
-
2465
- await this.publishSyncedEvents(storedStatus);
2466
- } finally {
2467
- this.connection.syncing = false;
2468
- this.syncing = false;
2469
- lock.release();
2470
- }
1812
+ return this.syncOps.partialSync(storedStatus);
2471
1813
  }
2472
1814
 
2473
1815
  /**
@@ -2592,7 +1934,7 @@ class Mailbox {
2592
1934
  }
2593
1935
 
2594
1936
  // Check for UIDVALIDITY change (mailbox recreated)
2595
- if ('uidValidity' in storedStatus && mailboxStatus.uidValidity !== storedStatus.uidValidity) {
1937
+ if (hasUidValidityChanged(storedStatus, mailboxStatus)) {
2596
1938
  // UIDVALIDITY has changed, full sync is required!
2597
1939
  // delete mailbox status
2598
1940
  let result = await this.connection.redis.multi().zcard(this.getMessagesKey()).del(this.getMessagesKey()).del(this.getMailboxKey()).exec();
@@ -2621,10 +1963,10 @@ class Mailbox {
2621
1963
  storedStatus = await this.getStoredStatus();
2622
1964
  }
2623
1965
 
2624
- // Determine sync strategy based on various conditions
1966
+ // Determine sync strategy using helper functions
2625
1967
 
2626
1968
  // No changes if MODSEQ hasn't changed
2627
- if (storedStatus.highestModseq && storedStatus.highestModseq === mailboxStatus.highestModseq) {
1969
+ if (hasNoModseqChanges(storedStatus, mailboxStatus)) {
2628
1970
  return false;
2629
1971
  }
2630
1972
 
@@ -2633,34 +1975,18 @@ class Mailbox {
2633
1975
  return false;
2634
1976
  }
2635
1977
 
2636
- // Partial sync if we can detect only new messages or flag changes
2637
- if (
2638
- this.connection.imapClient.enabled.has('CONDSTORE') &&
2639
- storedStatus.highestModseq < mailboxStatus.highestModseq &&
2640
- storedStatus.messages <= mailboxStatus.messages &&
2641
- mailboxStatus.uidNext - storedStatus.uidNext === mailboxStatus.messages - storedStatus.messages
2642
- ) {
2643
- // search for flag changes and new messages
1978
+ // Partial sync if CONDSTORE indicates only new messages or flag changes
1979
+ if (canUseCondstorePartialSync(this.connection.imapClient, storedStatus, mailboxStatus)) {
2644
1980
  return await this.partialSync(storedStatus);
2645
1981
  }
2646
1982
 
2647
1983
  // Partial sync if only new messages
2648
- if (
2649
- storedStatus.messages < mailboxStatus.messages &&
2650
- mailboxStatus.uidNext - storedStatus.uidNext === mailboxStatus.messages - storedStatus.messages
2651
- ) {
2652
- // seem to have new messages only
1984
+ if (canUseSimplePartialSync(storedStatus, mailboxStatus)) {
2653
1985
  return await this.partialSync(storedStatus);
2654
1986
  }
2655
1987
 
2656
1988
  // Skip if nothing changed and recent full sync
2657
- if (
2658
- storedStatus.messages === mailboxStatus.messages &&
2659
- storedStatus.uidNext === mailboxStatus.uidNext &&
2660
- storedStatus.lastFullSync &&
2661
- storedStatus.lastFullSync >= new Date(Date.now() - FULL_SYNC_DELAY)
2662
- ) {
2663
- // too soon from last full sync, message count seems the same
1989
+ if (canSkipSync(storedStatus, mailboxStatus)) {
2664
1990
  return false;
2665
1991
  }
2666
1992