imapflow 1.0.145 → 1.0.146

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (5) hide show
  1. package/CHANGELOG.md +7 -0
  2. package/lib/commands/fetch.js +152 -135
  3. package/lib/commands/idle.js +2 -0
  4. package/lib/imap-flow.js +22 -13
  5. package/package.json +1 -1
package/CHANGELOG.md CHANGED
@@ -1,5 +1,12 @@
1
1
  # Changelog
2
2
 
3
+ ## [1.0.146](https://github.com/postalsys/imapflow/compare/v1.0.145...v1.0.146) (2023-10-26)
4
+
5
+
6
+ ### Bug Fixes
7
+
8
+ * **throttling:** automatically retry throttled FETCH commands a few times ([07a9aea](https://github.com/postalsys/imapflow/commit/07a9aea37ae55ed885ccc4c0e3732b367ad82cd3))
9
+
3
10
  ## [1.0.145](https://github.com/postalsys/imapflow/compare/v1.0.144...v1.0.145) (2023-10-26)
4
11
 
5
12
 
package/lib/commands/fetch.js CHANGED
@@ -13,180 +13,197 @@ module.exports = async (connection, range, query, options) => {
13
13
 
14
14
  let mailbox = connection.mailbox;
15
15
 
16
- let messages = {
17
- count: 0,
18
- list: []
19
- };
20
-
21
16
  const commandKey = connection.capabilities.has('BINARY') && options.binary && !connection.disableBinary ? 'BINARY' : 'BODY';
22
17
 
23
- let response;
24
- try {
25
- let attributes = [{ type: 'SEQUENCE', value: (range || '*').toString() }];
18
+ let retryCount = 0;
19
+ while (retryCount < 4) {
20
+ let messages = {
21
+ count: 0,
22
+ list: []
23
+ };
26
24
 
27
- let queryStructure = [];
25
+ let response;
26
+ try {
27
+ let attributes = [{ type: 'SEQUENCE', value: (range || '*').toString() }];
28
28
 
29
- let setBodyPeek = (attributes, partial) => {
30
- let bodyPeek = {
31
- type: 'ATOM',
32
- value: `${commandKey}.PEEK`,
33
- section: [],
34
- partial
35
- };
29
+ let queryStructure = [];
36
30
 
37
- if (Array.isArray(attributes)) {
38
- attributes.forEach(attribute => {
39
- bodyPeek.section.push(attribute);
40
- });
41
- } else if (attributes) {
42
- bodyPeek.section.push(attributes);
43
- }
31
+ let setBodyPeek = (attributes, partial) => {
32
+ let bodyPeek = {
33
+ type: 'ATOM',
34
+ value: `${commandKey}.PEEK`,
35
+ section: [],
36
+ partial
37
+ };
38
+
39
+ if (Array.isArray(attributes)) {
40
+ attributes.forEach(attribute => {
41
+ bodyPeek.section.push(attribute);
42
+ });
43
+ } else if (attributes) {
44
+ bodyPeek.section.push(attributes);
45
+ }
44
46
 
45
- queryStructure.push(bodyPeek);
46
- };
47
+ queryStructure.push(bodyPeek);
48
+ };
47
49
 
48
- ['all', 'fast', 'full', 'uid', 'flags', 'bodyStructure', 'envelope', 'internalDate'].forEach(key => {
49
- if (query[key]) {
50
- queryStructure.push({ type: 'ATOM', value: key.toUpperCase() });
51
- }
52
- });
50
+ ['all', 'fast', 'full', 'uid', 'flags', 'bodyStructure', 'envelope', 'internalDate'].forEach(key => {
51
+ if (query[key]) {
52
+ queryStructure.push({ type: 'ATOM', value: key.toUpperCase() });
53
+ }
54
+ });
53
55
 
54
- if (query.size) {
55
- queryStructure.push({ type: 'ATOM', value: 'RFC822.SIZE' });
56
- }
56
+ if (query.size) {
57
+ queryStructure.push({ type: 'ATOM', value: 'RFC822.SIZE' });
58
+ }
57
59
 
58
- if (query.source) {
59
- let partial;
60
- if (typeof query.source === 'object' && (query.source.start || query.source.maxLength)) {
61
- partial = [Number(query.source.start) || 0];
62
- if (query.source.maxLength && !isNaN(query.source.maxLength)) {
63
- partial.push(Number(query.source.maxLength));
60
+ if (query.source) {
61
+ let partial;
62
+ if (typeof query.source === 'object' && (query.source.start || query.source.maxLength)) {
63
+ partial = [Number(query.source.start) || 0];
64
+ if (query.source.maxLength && !isNaN(query.source.maxLength)) {
65
+ partial.push(Number(query.source.maxLength));
66
+ }
64
67
  }
68
+ queryStructure.push({ type: 'ATOM', value: `${commandKey}.PEEK`, section: [], partial });
65
69
  }
66
- queryStructure.push({ type: 'ATOM', value: `${commandKey}.PEEK`, section: [], partial });
67
- }
68
70
 
69
- // if possible, always request for unique email id
70
- if (connection.capabilities.has('OBJECTID')) {
71
- queryStructure.push({ type: 'ATOM', value: 'EMAILID' });
72
- } else if (connection.capabilities.has('X-GM-EXT-1')) {
73
- queryStructure.push({ type: 'ATOM', value: 'X-GM-MSGID' });
74
- }
75
-
76
- if (query.threadId) {
71
+ // if possible, always request for unique email id
77
72
  if (connection.capabilities.has('OBJECTID')) {
78
- queryStructure.push({ type: 'ATOM', value: 'THREADID' });
73
+ queryStructure.push({ type: 'ATOM', value: 'EMAILID' });
79
74
  } else if (connection.capabilities.has('X-GM-EXT-1')) {
80
- queryStructure.push({ type: 'ATOM', value: 'X-GM-THRID' });
75
+ queryStructure.push({ type: 'ATOM', value: 'X-GM-MSGID' });
81
76
  }
82
- }
83
77
 
84
- if (query.labels) {
85
- if (connection.capabilities.has('X-GM-EXT-1')) {
86
- queryStructure.push({ type: 'ATOM', value: 'X-GM-LABELS' });
78
+ if (query.threadId) {
79
+ if (connection.capabilities.has('OBJECTID')) {
80
+ queryStructure.push({ type: 'ATOM', value: 'THREADID' });
81
+ } else if (connection.capabilities.has('X-GM-EXT-1')) {
82
+ queryStructure.push({ type: 'ATOM', value: 'X-GM-THRID' });
83
+ }
87
84
  }
88
- }
89
85
 
90
- // always ask for modseq if possible
91
- if (connection.enabled.has('CONDSTORE') && !mailbox.noModseq) {
92
- queryStructure.push({ type: 'ATOM', value: 'MODSEQ' });
93
- }
86
+ if (query.labels) {
87
+ if (connection.capabilities.has('X-GM-EXT-1')) {
88
+ queryStructure.push({ type: 'ATOM', value: 'X-GM-LABELS' });
89
+ }
90
+ }
94
91
 
95
- // always make sure to include UID in the request as well even though server might auto-add it itself
96
- if (!query.uid) {
97
- queryStructure.push({ type: 'ATOM', value: 'UID' });
98
- }
92
+ // always ask for modseq if possible
93
+ if (connection.enabled.has('CONDSTORE') && !mailbox.noModseq) {
94
+ queryStructure.push({ type: 'ATOM', value: 'MODSEQ' });
95
+ }
99
96
 
100
- if (query.headers) {
101
- if (Array.isArray(query.headers)) {
102
- setBodyPeek([{ type: 'ATOM', value: 'HEADER.FIELDS' }, query.headers.map(header => ({ type: 'ATOM', value: header }))]);
103
- } else {
104
- setBodyPeek({ type: 'ATOM', value: 'HEADER' });
97
+ // always make sure to include UID in the request as well even though server might auto-add it itself
98
+ if (!query.uid) {
99
+ queryStructure.push({ type: 'ATOM', value: 'UID' });
105
100
  }
106
- }
107
101
 
108
- if (query.bodyParts && query.bodyParts.length) {
109
- query.bodyParts.forEach(part => {
110
- if (!part) {
111
- return;
102
+ if (query.headers) {
103
+ if (Array.isArray(query.headers)) {
104
+ setBodyPeek([{ type: 'ATOM', value: 'HEADER.FIELDS' }, query.headers.map(header => ({ type: 'ATOM', value: header }))]);
105
+ } else {
106
+ setBodyPeek({ type: 'ATOM', value: 'HEADER' });
112
107
  }
113
- let key;
114
- let partial;
115
- if (typeof part === 'object') {
116
- if (!part.key || typeof part.key !== 'string') {
108
+ }
109
+
110
+ if (query.bodyParts && query.bodyParts.length) {
111
+ query.bodyParts.forEach(part => {
112
+ if (!part) {
117
113
  return;
118
114
  }
119
- key = part.key.toUpperCase();
120
- if (part.start || part.maxLength) {
121
- partial = [Number(part.start) || 0];
122
- if (part.maxLength && !isNaN(part.maxLength)) {
123
- partial.push(Number(part.maxLength));
115
+ let key;
116
+ let partial;
117
+ if (typeof part === 'object') {
118
+ if (!part.key || typeof part.key !== 'string') {
119
+ return;
124
120
  }
121
+ key = part.key.toUpperCase();
122
+ if (part.start || part.maxLength) {
123
+ partial = [Number(part.start) || 0];
124
+ if (part.maxLength && !isNaN(part.maxLength)) {
125
+ partial.push(Number(part.maxLength));
126
+ }
127
+ }
128
+ } else if (typeof part === 'string') {
129
+ key = part.toUpperCase();
130
+ } else {
131
+ return;
125
132
  }
126
- } else if (typeof part === 'string') {
127
- key = part.toUpperCase();
128
- } else {
129
- return;
130
- }
131
133
 
132
- setBodyPeek({ type: 'ATOM', value: key }, partial);
133
- });
134
- }
134
+ setBodyPeek({ type: 'ATOM', value: key }, partial);
135
+ });
136
+ }
135
137
 
136
- if (queryStructure.length === 1) {
137
- queryStructure = queryStructure.pop();
138
- }
138
+ if (queryStructure.length === 1) {
139
+ queryStructure = queryStructure.pop();
140
+ }
139
141
 
140
- attributes.push(queryStructure);
142
+ attributes.push(queryStructure);
143
+
144
+ if (options.changedSince && connection.enabled.has('CONDSTORE') && !mailbox.noModseq) {
145
+ let changedSinceArgs = [
146
+ {
147
+ type: 'ATOM',
148
+ value: 'CHANGEDSINCE'
149
+ },
150
+ {
151
+ type: 'ATOM',
152
+ value: options.changedSince.toString()
153
+ }
154
+ ];
141
155
 
142
- if (options.changedSince && connection.enabled.has('CONDSTORE') && !mailbox.noModseq) {
143
- let changedSinceArgs = [
144
- {
145
- type: 'ATOM',
146
- value: 'CHANGEDSINCE'
147
- },
148
- {
149
- type: 'ATOM',
150
- value: options.changedSince.toString()
156
+ if (options.uid && connection.enabled.has('QRESYNC')) {
157
+ changedSinceArgs.push({
158
+ type: 'ATOM',
159
+ value: 'VANISHED'
160
+ });
151
161
  }
152
- ];
153
162
 
154
- if (options.uid && connection.enabled.has('QRESYNC')) {
155
- changedSinceArgs.push({
156
- type: 'ATOM',
157
- value: 'VANISHED'
158
- });
163
+ attributes.push(changedSinceArgs);
159
164
  }
160
165
 
161
- attributes.push(changedSinceArgs);
162
- }
163
-
164
- response = await connection.exec(options.uid ? 'UID FETCH' : 'FETCH', attributes, {
165
- untagged: {
166
- FETCH: async untagged => {
167
- messages.count++;
168
- let formatted = await formatMessageResponse(untagged, mailbox);
169
- if (typeof options.onUntaggedFetch === 'function') {
170
- await new Promise((resolve, reject) => {
171
- options.onUntaggedFetch(formatted, err => {
172
- if (err) {
173
- reject(err);
174
- } else {
175
- resolve();
176
- }
166
+ response = await connection.exec(options.uid ? 'UID FETCH' : 'FETCH', attributes, {
167
+ untagged: {
168
+ FETCH: async untagged => {
169
+ messages.count++;
170
+ let formatted = await formatMessageResponse(untagged, mailbox);
171
+ if (typeof options.onUntaggedFetch === 'function') {
172
+ await new Promise((resolve, reject) => {
173
+ options.onUntaggedFetch(formatted, err => {
174
+ if (err) {
175
+ reject(err);
176
+ } else {
177
+ resolve();
178
+ }
179
+ });
177
180
  });
178
- });
179
- } else {
180
- messages.list.push(formatted);
181
+ } else {
182
+ messages.list.push(formatted);
183
+ }
181
184
  }
182
185
  }
186
+ });
187
+
188
+ response.next();
189
+ return messages;
190
+ } catch (err) {
191
+ if (err.code === 'ETHROTTLE') {
192
+ // retrying
193
+ connection.log.warn({
194
+ msg: 'Retrying throttled request',
195
+ cid: connection.id,
196
+ code: err.code,
197
+ response: err.responseText,
198
+ throttleReset: err.throttleReset,
199
+ retryCount
200
+ });
201
+ retryCount++;
202
+ continue;
183
203
  }
184
- });
185
204
 
186
- response.next();
187
- return messages;
188
- } catch (err) {
189
- connection.log.warn({ err, cid: connection.id });
190
- return false;
205
+ connection.log.warn({ err, cid: connection.id });
206
+ return false;
207
+ }
191
208
  }
192
209
  };
package/lib/commands/idle.js CHANGED
@@ -110,6 +110,8 @@ module.exports = async (connection, maxIdleTime) => {
110
110
  return;
111
111
  }
112
112
 
113
+ maxIdleTime = 3000;
114
+
113
115
  if (connection.capabilities.has('IDLE')) {
114
116
  let idleTimer;
115
117
  let stillIdling = false;
package/lib/imap-flow.js CHANGED
@@ -562,25 +562,31 @@ class ImapFlow extends EventEmitter {
562
562
  if (txt) {
563
563
  err.responseText = txt;
564
564
 
565
+ let throttleDelay = false;
566
+
565
567
  // MS365 throttling
566
568
  // tag BAD Request is throttled. Suggested Backoff Time: 92415 milliseconds
567
569
  if (/Request is throttled/i.test(txt) && /Backoff Time/i.test(txt)) {
568
570
  let throttlingMatch = txt.match(/Backoff Time[:=\s]+(\d+)/i);
569
571
  if (throttlingMatch && throttlingMatch[1] && !isNaN(throttlingMatch[1])) {
570
- err.code = 'ETHROTTLE';
571
- err.throttleReset = Number(throttlingMatch[1]);
572
-
573
- let delayResponse = err.throttleReset;
574
- if (delayResponse > 5 * 60 * 1000) {
575
- // max delay cap
576
- delayResponse = 5 * 60 * 1000;
577
- }
578
- if (delayResponse) {
579
- this.log.warn({ msg: 'Throttling detected', err, cid: this.id, delayResponse });
580
- await new Promise(r => setTimeout(r, delayResponse));
581
- }
572
+ throttleDelay = Number(throttlingMatch[1]);
582
573
  }
583
574
  }
575
+
576
+ // Wait and return a throttling error
577
+ if (throttleDelay) {
578
+ err.code = 'ETHROTTLE';
579
+ err.throttleReset = throttleDelay;
580
+
581
+ let delayResponse = throttleDelay;
582
+ if (delayResponse > 5 * 60 * 1000) {
583
+ // max delay cap
584
+ delayResponse = 5 * 60 * 1000;
585
+ }
586
+
587
+ this.log.warn({ msg: 'Throttling detected', err, cid: this.id, throttleDelay, delayResponse });
588
+ await new Promise(r => setTimeout(r, delayResponse));
589
+ }
584
590
  }
585
591
 
586
592
  request.reject(err);
@@ -2736,7 +2742,10 @@ class ImapFlow extends EventEmitter {
2736
2742
 
2737
2743
  let result = await handler(this, ...args);
2738
2744
 
2739
- this.autoidle();
2745
+ if (command !== 'IDLE') {
2746
+ // do not autostart IDLE, if IDLE itself was stopped
2747
+ this.autoidle();
2748
+ }
2740
2749
 
2741
2750
  return result;
2742
2751
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "imapflow",
3
- "version": "1.0.145",
3
+ "version": "1.0.146",
4
4
  "description": "IMAP Client for Node",
5
5
  "main": "./lib/imap-flow.js",
6
6
  "scripts": {