imapflow 2.2.6 → 2.2.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/dist/cjs/commands/append.js +10 -3
- package/dist/cjs/commands/authenticate.js +42 -22
- package/dist/cjs/commands/close.d.ts +12 -1
- package/dist/cjs/commands/close.js +4 -2
- package/dist/cjs/commands/delete.js +2 -1
- package/dist/cjs/commands/enable.js +6 -0
- package/dist/cjs/commands/esearch-parser.js +8 -2
- package/dist/cjs/commands/fetch.js +57 -14
- package/dist/cjs/commands/id.js +8 -1
- package/dist/cjs/commands/idle.js +15 -5
- package/dist/cjs/commands/list.js +10 -1
- package/dist/cjs/commands/login.js +5 -1
- package/dist/cjs/commands/logout.js +7 -0
- package/dist/cjs/commands/namespace.js +7 -3
- package/dist/cjs/commands/quota.js +3 -1
- package/dist/cjs/commands/rename.js +2 -1
- package/dist/cjs/commands/select.js +5 -0
- package/dist/cjs/commands/status.js +6 -1
- package/dist/cjs/commands/store.d.ts +1 -1
- package/dist/cjs/commands/store.js +8 -8
- package/dist/cjs/download.js +220 -95
- package/dist/cjs/handler/imap-compiler.js +19 -10
- package/dist/cjs/handler/limits.d.ts +11 -0
- package/dist/cjs/handler/limits.js +16 -1
- package/dist/cjs/handler/parser-instance.d.ts +10 -0
- package/dist/cjs/handler/parser-instance.js +25 -10
- package/dist/cjs/handler/token-parser.js +36 -28
- package/dist/cjs/imap-flow.d.ts +2 -2
- package/dist/cjs/imap-flow.js +256 -95
- package/dist/cjs/package-info.d.ts +1 -1
- package/dist/cjs/package-info.js +1 -1
- package/dist/cjs/proxy-connection.js +7 -7
- package/dist/cjs/search-compiler.js +33 -13
- package/dist/cjs/special-use.js +10 -5
- package/dist/cjs/tools.d.ts +14 -4
- package/dist/cjs/tools.js +69 -9
- package/dist/cjs/types.d.ts +19 -4
- package/dist/esm/commands/append.js +11 -4
- package/dist/esm/commands/authenticate.js +43 -23
- package/dist/esm/commands/close.d.ts +12 -1
- package/dist/esm/commands/close.js +5 -3
- package/dist/esm/commands/delete.js +2 -1
- package/dist/esm/commands/enable.js +6 -0
- package/dist/esm/commands/esearch-parser.js +8 -2
- package/dist/esm/commands/fetch.js +57 -14
- package/dist/esm/commands/id.js +9 -2
- package/dist/esm/commands/idle.js +16 -6
- package/dist/esm/commands/list.js +10 -1
- package/dist/esm/commands/login.js +6 -2
- package/dist/esm/commands/logout.js +7 -0
- package/dist/esm/commands/namespace.js +7 -3
- package/dist/esm/commands/quota.js +3 -1
- package/dist/esm/commands/rename.js +2 -1
- package/dist/esm/commands/select.js +5 -0
- package/dist/esm/commands/status.js +7 -2
- package/dist/esm/commands/store.d.ts +1 -1
- package/dist/esm/commands/store.js +9 -9
- package/dist/esm/download.js +220 -95
- package/dist/esm/handler/imap-compiler.js +19 -10
- package/dist/esm/handler/limits.d.ts +11 -0
- package/dist/esm/handler/limits.js +14 -0
- package/dist/esm/handler/parser-instance.d.ts +10 -0
- package/dist/esm/handler/parser-instance.js +25 -10
- package/dist/esm/handler/token-parser.js +37 -29
- package/dist/esm/imap-flow.d.ts +2 -2
- package/dist/esm/imap-flow.js +256 -95
- package/dist/esm/package-info.d.ts +1 -1
- package/dist/esm/package-info.js +1 -1
- package/dist/esm/proxy-connection.js +7 -7
- package/dist/esm/search-compiler.js +33 -13
- package/dist/esm/special-use.js +10 -5
- package/dist/esm/tools.d.ts +14 -4
- package/dist/esm/tools.js +68 -9
- package/dist/esm/types.d.ts +19 -4
- package/package.json +5 -3
package/dist/esm/download.js
CHANGED
|
@@ -8,7 +8,117 @@ import libbase64 from 'libbase64';
|
|
|
8
8
|
import { Headers } from '@zone-eu/mailsplit';
|
|
9
9
|
import FlowedDecoder from '@zone-eu/mailsplit/lib/flowed-decoder.js';
|
|
10
10
|
import { LimitedPassthrough, normalizeByteLimit } from './limited-passthrough.js';
|
|
11
|
+
// What a wait on the head stream listens for: it can take more input, it failed, or it went
|
|
12
|
+
// away (a consumer destroying it closes it without a 'drain')
|
|
13
|
+
const DRAIN_WAIT_EVENTS = ['drain', 'error', 'close'];
|
|
11
14
|
import { getDecoder, isUnsafeKey } from './tools.js';
|
|
15
|
+
const isEmptySection = (value) => !value?.length;
|
|
16
|
+
/**
|
|
17
|
+
* The section to ask again when exactly one of a part's MIME headers and content came back
|
|
18
|
+
* empty (see refetchDroppedSections()), undefined when both or neither did
|
|
19
|
+
*/
|
|
20
|
+
const droppedSection = (mime, content, mimeRequest, contentRequest) => {
|
|
21
|
+
if (isEmptySection(mime) === isEmptySection(content)) {
|
|
22
|
+
return undefined;
|
|
23
|
+
}
|
|
24
|
+
return isEmptySection(content) ? contentRequest : mimeRequest;
|
|
25
|
+
};
|
|
26
|
+
/** Start offset of every partial section among the requests, keyed by section */
|
|
27
|
+
const partialStarts = (requests) => {
|
|
28
|
+
let starts = new Map();
|
|
29
|
+
for (let request of requests) {
|
|
30
|
+
if (typeof request !== 'string') {
|
|
31
|
+
starts.set(request.key, Number(request.start) || 0);
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
return starts;
|
|
35
|
+
};
|
|
36
|
+
// How many times a request is repeated after answers that belong to another request
|
|
37
|
+
const MAX_FOREIGN_ANSWERS = 3;
|
|
38
|
+
const requestedUid = (range, options) => options.uid && /^\d+$/.test(String(range)) ? Number(range) : undefined;
|
|
39
|
+
const isForeignAnswer = (response, expected) => {
|
|
40
|
+
if (expected.uid && response.uid && response.uid !== expected.uid) {
|
|
41
|
+
return true;
|
|
42
|
+
}
|
|
43
|
+
for (let [key, start] of expected.origins) {
|
|
44
|
+
let origin = response.partialOrigins && response.partialOrigins.get(key);
|
|
45
|
+
// An answer without the origin is taken as is: some servers leave it out, and some
|
|
46
|
+
// ignore the partial specifier altogether
|
|
47
|
+
if (typeof origin === 'number' && origin !== start) {
|
|
48
|
+
return true;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
return false;
|
|
52
|
+
};
|
|
53
|
+
/**
|
|
54
|
+
* fetchOne() that takes only an answer belonging to the request. Apache James now and then
|
|
55
|
+
* writes the head of a FETCH answer after its tagged OK, so the data of one request shows up
|
|
56
|
+
* within the answer to the next one. Taken at face value it would be the data of that next
|
|
57
|
+
* request, and a download would end without an error but with misplaced bytes. An answer for
|
|
58
|
+
* another UID, or with a partial section that starts at another offset than asked, is dropped
|
|
59
|
+
* and the request repeated.
|
|
60
|
+
*/
|
|
61
|
+
async function fetchExpected(client, range, query, options, expected) {
|
|
62
|
+
for (let attempt = 1;; attempt++) {
|
|
63
|
+
let response = await client.fetchOne(range, query, options);
|
|
64
|
+
if (!response || !isForeignAnswer(response, expected)) {
|
|
65
|
+
return response;
|
|
66
|
+
}
|
|
67
|
+
client.log.warn({
|
|
68
|
+
msg: 'Server answered with data of another request, asking again',
|
|
69
|
+
uid: response.uid,
|
|
70
|
+
origins: response.partialOrigins && Object.fromEntries(response.partialOrigins),
|
|
71
|
+
attempt,
|
|
72
|
+
cid: client.id
|
|
73
|
+
});
|
|
74
|
+
if (attempt >= MAX_FOREIGN_ANSWERS) {
|
|
75
|
+
let err = new Error('Server kept answering with data of another request');
|
|
76
|
+
err.code = 'DownloadIncomplete';
|
|
77
|
+
err.cid = client.id;
|
|
78
|
+
throw err;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Apache James (and the servers built on it, Twake Mail among them) answers only the first
|
|
84
|
+
* section it is asked for each MIME part of one FETCH and returns the other one empty:
|
|
85
|
+
* BODY[2.MIME] with BODY[2] yields the headers and a zero-length body, the reverse order loses
|
|
86
|
+
* the headers (FetchGroup.addPartContent() keeps the first descriptor for a part path). Asks the
|
|
87
|
+
* sections that came back empty again, in a FETCH of their own, and merges the answer into
|
|
88
|
+
* `response`. Callers only list a section whose companion did arrive, so a compliant server
|
|
89
|
+
* pays the extra round trip only for a part that really is empty.
|
|
90
|
+
*/
|
|
91
|
+
async function refetchDroppedSections(client, response, range, options, sections) {
|
|
92
|
+
if (!sections.length) {
|
|
93
|
+
return;
|
|
94
|
+
}
|
|
95
|
+
client.log.debug({
|
|
96
|
+
msg: 'Server answered a body section empty while its companion section was not, asking it again separately',
|
|
97
|
+
sections: sections.map(section => (typeof section === 'string' ? section : section.key)),
|
|
98
|
+
cid: client.id
|
|
99
|
+
});
|
|
100
|
+
// the UID pins the message even when the first command addressed it by sequence number
|
|
101
|
+
let uid = response.uid;
|
|
102
|
+
let retry = await fetchExpected(client, uid || range, { uid: true, bodyParts: sections }, uid ? { ...options, uid: true } : options, {
|
|
103
|
+
uid,
|
|
104
|
+
origins: partialStarts(sections)
|
|
105
|
+
});
|
|
106
|
+
if (!retry) {
|
|
107
|
+
return;
|
|
108
|
+
}
|
|
109
|
+
if (retry.headers) {
|
|
110
|
+
response.headers = retry.headers;
|
|
111
|
+
}
|
|
112
|
+
for (let [key, value] of retry.bodyParts || []) {
|
|
113
|
+
(response.bodyParts ??= new Map()).set(key, value);
|
|
114
|
+
if (retry.binaryParts && retry.binaryParts.has(key)) {
|
|
115
|
+
(response.binaryParts ??= new Set()).add(key);
|
|
116
|
+
}
|
|
117
|
+
else if (response.binaryParts) {
|
|
118
|
+
response.binaryParts.delete(key);
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
}
|
|
12
122
|
/**
|
|
13
123
|
* Implements ImapFlow.download(), see its documentation
|
|
14
124
|
*
|
|
@@ -47,7 +157,8 @@ export async function downloadMessage(client, range, part, options) {
|
|
|
47
157
|
range = uid;
|
|
48
158
|
downloadOptions.uid = true;
|
|
49
159
|
}
|
|
50
|
-
|
|
160
|
+
// bodyStructure is unset when the server sent BODYSTRUCTURE NIL
|
|
161
|
+
if (!response.bodyStructure?.childNodes) {
|
|
51
162
|
// single text message
|
|
52
163
|
part = 'TEXT';
|
|
53
164
|
}
|
|
@@ -55,6 +166,7 @@ export async function downloadMessage(client, range, part, options) {
|
|
|
55
166
|
let getNextPart = async (query) => {
|
|
56
167
|
query = query || {};
|
|
57
168
|
let mimeKey;
|
|
169
|
+
let contentRequest;
|
|
58
170
|
if (!part) {
|
|
59
171
|
query.source = {
|
|
60
172
|
start: processed,
|
|
@@ -77,13 +189,18 @@ export async function downloadMessage(client, range, part, options) {
|
|
|
77
189
|
query.bodyParts.push(mimeKey);
|
|
78
190
|
}
|
|
79
191
|
}
|
|
80
|
-
|
|
192
|
+
contentRequest = {
|
|
81
193
|
key: part,
|
|
82
194
|
start: processed,
|
|
83
195
|
maxLength: chunkSize
|
|
84
|
-
}
|
|
196
|
+
};
|
|
197
|
+
query.bodyParts.push(contentRequest);
|
|
85
198
|
}
|
|
86
|
-
let
|
|
199
|
+
let expected = {
|
|
200
|
+
uid: uid || requestedUid(range, downloadOptions),
|
|
201
|
+
origins: new Map([[part || '', processed]])
|
|
202
|
+
};
|
|
203
|
+
let response = await fetchExpected(client, range, query, downloadOptions, expected);
|
|
87
204
|
if (!response) {
|
|
88
205
|
return { response: false, chunk: false };
|
|
89
206
|
}
|
|
@@ -93,6 +210,12 @@ export async function downloadMessage(client, range, part, options) {
|
|
|
93
210
|
range = uid;
|
|
94
211
|
downloadOptions.uid = true;
|
|
95
212
|
}
|
|
213
|
+
if (mimeKey && contentRequest) {
|
|
214
|
+
let dropped = droppedSection(mimeKey === 'header' ? response.headers : response.bodyParts?.get(mimeKey), response.bodyParts?.get(part), mimeKey, contentRequest);
|
|
215
|
+
if (dropped) {
|
|
216
|
+
await refetchDroppedSections(client, response, range, downloadOptions, [dropped]);
|
|
217
|
+
}
|
|
218
|
+
}
|
|
96
219
|
let chunk = !part ? response.source : response.bodyParts && response.bodyParts.get(part);
|
|
97
220
|
if (!chunk) {
|
|
98
221
|
return {};
|
|
@@ -290,11 +413,54 @@ export async function downloadMessage(client, range, part, options) {
|
|
|
290
413
|
// at the bound still gets its terminating chunk. Infinity when the server reported no
|
|
291
414
|
// size, which leaves the loop bounded by maxBytes alone.
|
|
292
415
|
let maxTotalBytes = normalizeByteLimit(meta.expectedSize ? meta.expectedSize * 2 + chunkSize : 0);
|
|
293
|
-
//
|
|
294
|
-
//
|
|
295
|
-
//
|
|
296
|
-
//
|
|
297
|
-
|
|
416
|
+
// Resolves once the head stream can take more input, rejects when it fails, and resolves
|
|
417
|
+
// when it goes away. finish() is the listener itself: 'drain' and 'close' emit no arguments,
|
|
418
|
+
// 'error' emits the error, and removal needs no separate handler references. It removes
|
|
419
|
+
// only the listeners this wait installed - removeAllListeners('error') also took off the
|
|
420
|
+
// forwarder pipeStage() attached to the head stream when the pipeline was built, and the
|
|
421
|
+
// head must keep that forwarder for the life of the download or a chunk failure has nowhere
|
|
422
|
+
// to go.
|
|
423
|
+
let waitForDrain = () => new Promise((resolve, reject) => {
|
|
424
|
+
const finish = (err) => {
|
|
425
|
+
for (let event of DRAIN_WAIT_EVENTS) {
|
|
426
|
+
stream.removeListener(event, finish);
|
|
427
|
+
}
|
|
428
|
+
/* c8 ignore next 2 */ // stream error during a backpressure drain wait is timing-dependent
|
|
429
|
+
if (err) {
|
|
430
|
+
reject(err);
|
|
431
|
+
}
|
|
432
|
+
else {
|
|
433
|
+
resolve();
|
|
434
|
+
}
|
|
435
|
+
};
|
|
436
|
+
for (let event of DRAIN_WAIT_EVENTS) {
|
|
437
|
+
stream.once(event, finish);
|
|
438
|
+
}
|
|
439
|
+
});
|
|
440
|
+
// Writes a chunk and waits out the backpressure. A base64 decoder defers its write callback,
|
|
441
|
+
// so a chunk the size of its buffer is never taken synchronously and the head chunk waits
|
|
442
|
+
// here as much as any other. A stream failure during the wait is thrown unless the download
|
|
443
|
+
// was already aborted (the consumer destroyed the stream).
|
|
444
|
+
let writeAndDrain = async (chunk) => {
|
|
445
|
+
if (writeChunk(chunk) !== false) {
|
|
446
|
+
return;
|
|
447
|
+
}
|
|
448
|
+
try {
|
|
449
|
+
await waitForDrain();
|
|
450
|
+
/* c8 ignore next 5 */ // re-throw path only triggers on a stream error mid-drain, which is timing-dependent
|
|
451
|
+
}
|
|
452
|
+
catch (err) {
|
|
453
|
+
if (!fetchAborted) {
|
|
454
|
+
throw err;
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
};
|
|
458
|
+
// Writes the head chunk, then fetches the remaining chunks in a loop, writing each to the
|
|
459
|
+
// decoder stream. Stops when the server returns a short chunk (< chunkSize), answers with
|
|
460
|
+
// more than the requested window, the byte limiter is satisfied, or the consumer destroys
|
|
461
|
+
// the output stream. Throws when the ceiling above is crossed.
|
|
462
|
+
let fetchAllParts = async (head) => {
|
|
463
|
+
await writeAndDrain(head);
|
|
298
464
|
while (hasMore && !isLimited() && !fetchAborted) {
|
|
299
465
|
if (processed >= maxTotalBytes) {
|
|
300
466
|
// Loud on purpose. Everything written downstream by this point holds
|
|
@@ -307,7 +473,9 @@ export async function downloadMessage(client, range, part, options) {
|
|
|
307
473
|
throw err;
|
|
308
474
|
}
|
|
309
475
|
let { response, chunk } = await getNextPart();
|
|
310
|
-
|
|
476
|
+
// A consumer that gave up while the chunk was in flight: its 'close' may still be a
|
|
477
|
+
// tick away from setting fetchAborted, so the stream's own flag is checked as well
|
|
478
|
+
if (fetchAborted || output.destroyed) {
|
|
311
479
|
break;
|
|
312
480
|
}
|
|
313
481
|
if (response === false) {
|
|
@@ -322,48 +490,7 @@ export async function downloadMessage(client, range, part, options) {
|
|
|
322
490
|
if (!chunk) {
|
|
323
491
|
break;
|
|
324
492
|
}
|
|
325
|
-
|
|
326
|
-
if (writeChunk(chunk) === false) {
|
|
327
|
-
// Wait for drain event before continuing
|
|
328
|
-
try {
|
|
329
|
-
await new Promise((resolve, reject) => {
|
|
330
|
-
// finish() is the listener itself, as settle() is for the TLS upgrade:
|
|
331
|
-
// 'drain' and 'close' emit no arguments, 'error' emits the error, and
|
|
332
|
-
// removal needs no separate handler references. It removes only the
|
|
333
|
-
// three listeners this wait installed - removeAllListeners('error')
|
|
334
|
-
// also took off the forwarder pipeStage() attached to the head stream
|
|
335
|
-
// when the pipeline was built, and the head must keep that forwarder
|
|
336
|
-
// for the life of the download or a chunk failure has nowhere to go.
|
|
337
|
-
const finish = (err) => {
|
|
338
|
-
for (let event of ['drain', 'error', 'close']) {
|
|
339
|
-
stream.removeListener(event, finish);
|
|
340
|
-
}
|
|
341
|
-
/* c8 ignore next 2 */ // stream error during a backpressure drain wait is timing-dependent
|
|
342
|
-
if (err) {
|
|
343
|
-
reject(err);
|
|
344
|
-
}
|
|
345
|
-
else {
|
|
346
|
-
resolve();
|
|
347
|
-
}
|
|
348
|
-
};
|
|
349
|
-
stream.once('drain', finish);
|
|
350
|
-
stream.once('error', finish);
|
|
351
|
-
stream.once('close', finish);
|
|
352
|
-
});
|
|
353
|
-
/* c8 ignore start */ // re-throw path only triggers on a stream error mid-drain, which is timing-dependent
|
|
354
|
-
}
|
|
355
|
-
catch (err) {
|
|
356
|
-
// Re-throw only if not aborted
|
|
357
|
-
if (!fetchAborted) {
|
|
358
|
-
throw err;
|
|
359
|
-
}
|
|
360
|
-
}
|
|
361
|
-
/* c8 ignore stop */
|
|
362
|
-
// Check if we should abort after waiting
|
|
363
|
-
if (fetchAborted) {
|
|
364
|
-
break;
|
|
365
|
-
}
|
|
366
|
-
}
|
|
493
|
+
await writeAndDrain(chunk);
|
|
367
494
|
}
|
|
368
495
|
};
|
|
369
496
|
// A download is a sequence of chunk FETCHes with a backpressure wait in between. Those
|
|
@@ -382,13 +509,10 @@ export async function downloadMessage(client, range, part, options) {
|
|
|
382
509
|
client.autoidle();
|
|
383
510
|
}
|
|
384
511
|
};
|
|
385
|
-
//
|
|
386
|
-
//
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
// value before streaming begins.
|
|
390
|
-
let runFetchAllParts = () => {
|
|
391
|
-
fetchAllParts()
|
|
512
|
+
// Runs the download pipeline with the head chunk fetched above (for its metadata): it is
|
|
513
|
+
// written to the decoder stream and the remaining chunks follow
|
|
514
|
+
let runFetchAllParts = (head) => {
|
|
515
|
+
fetchAllParts(head)
|
|
392
516
|
.catch(err => {
|
|
393
517
|
if (!fetchAborted && stream && !stream.destroyed) {
|
|
394
518
|
stream.emit('error', err);
|
|
@@ -420,36 +544,8 @@ export async function downloadMessage(client, range, part, options) {
|
|
|
420
544
|
// for a routine-looking connection code.
|
|
421
545
|
.catch(err => client.log.error({ msg: 'Failed to fail the download stream', err, cid: client.id }));
|
|
422
546
|
};
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
try {
|
|
426
|
-
writeResult = writeChunk(chunk);
|
|
427
|
-
}
|
|
428
|
-
catch (err) {
|
|
429
|
-
stream.emit('error', err);
|
|
430
|
-
finishDownload();
|
|
431
|
-
/* c8 ignore next 3 */ // emitting the error above triggers cleanup (fetchAborted=true), so this end() guard is already false here
|
|
432
|
-
if (!fetchAborted && stream && !stream.destroyed) {
|
|
433
|
-
stream.end();
|
|
434
|
-
}
|
|
435
|
-
return;
|
|
436
|
-
}
|
|
437
|
-
/* c8 ignore next 9 */ // `stream` is piped to the limiter before this runs, so the head write drains synchronously and always returns true (verified for chunkSize up to 8MB); the drain-wait branch is unreachable
|
|
438
|
-
if (!writeResult) {
|
|
439
|
-
// Initial chunk filled the buffer, wait for drain
|
|
440
|
-
stream.once('drain', () => {
|
|
441
|
-
if (!fetchAborted) {
|
|
442
|
-
runFetchAllParts();
|
|
443
|
-
}
|
|
444
|
-
else {
|
|
445
|
-
finishDownload();
|
|
446
|
-
}
|
|
447
|
-
});
|
|
448
|
-
}
|
|
449
|
-
else {
|
|
450
|
-
runFetchAllParts();
|
|
451
|
-
}
|
|
452
|
-
});
|
|
547
|
+
// Deferred so the caller gets the {meta, content} return value before streaming begins
|
|
548
|
+
setImmediate(() => runFetchAllParts(chunk));
|
|
453
549
|
return {
|
|
454
550
|
meta,
|
|
455
551
|
content: output
|
|
@@ -469,19 +565,36 @@ export async function downloadMessageParts(client, range, parts, options) {
|
|
|
469
565
|
// no mailbox selected, nothing to do
|
|
470
566
|
return {};
|
|
471
567
|
}
|
|
472
|
-
let downloadOptions =
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
568
|
+
let downloadOptions = options || {};
|
|
569
|
+
// Asked as a partial fetch so at most maxBytes of each part crosses the wire, and enforced
|
|
570
|
+
// again on the answer for servers that ignore the partial specifier
|
|
571
|
+
let maxBytes = normalizeByteLimit(downloadOptions.maxBytes);
|
|
476
572
|
let query = { bodyParts: [] };
|
|
573
|
+
let contentRequests = new Map();
|
|
477
574
|
for (let part of parts) {
|
|
478
575
|
query.bodyParts.push(part + '.mime');
|
|
479
|
-
|
|
576
|
+
// The partial specifier carries a 32-bit length (RFC 9051 "number"), so a cap beyond
|
|
577
|
+
// that is applied on the answer alone
|
|
578
|
+
let contentRequest = maxBytes > 0xffffffff ? part : { key: part, start: 0, maxLength: maxBytes };
|
|
579
|
+
contentRequests.set(part, contentRequest);
|
|
580
|
+
query.bodyParts.push(contentRequest);
|
|
480
581
|
}
|
|
481
|
-
let response = await client
|
|
582
|
+
let response = await fetchExpected(client, range, query, downloadOptions, {
|
|
583
|
+
uid: requestedUid(range, downloadOptions),
|
|
584
|
+
origins: partialStarts(query.bodyParts)
|
|
585
|
+
});
|
|
482
586
|
if (!response || !response.bodyParts) {
|
|
483
587
|
return {};
|
|
484
588
|
}
|
|
589
|
+
let dropped = [];
|
|
590
|
+
for (let [part, contentRequest] of contentRequests) {
|
|
591
|
+
let section = droppedSection(response.bodyParts.get(part + '.mime'), response.bodyParts.get(part), part + '.mime', contentRequest);
|
|
592
|
+
if (section) {
|
|
593
|
+
dropped.push(section);
|
|
594
|
+
}
|
|
595
|
+
}
|
|
596
|
+
// Sections of different parts do not collide, so every dropped one fits in one FETCH
|
|
597
|
+
await refetchDroppedSections(client, response, range, downloadOptions, dropped);
|
|
485
598
|
let data = {};
|
|
486
599
|
for (let [part, content] of response.bodyParts) {
|
|
487
600
|
let keyParts = part.split('.mime');
|
|
@@ -494,6 +607,18 @@ export async function downloadMessageParts(client, range, parts, options) {
|
|
|
494
607
|
if (keyParts.length === 1) {
|
|
495
608
|
// content
|
|
496
609
|
let key = keyParts[0];
|
|
610
|
+
if (content.length > maxBytes) {
|
|
611
|
+
// The server ignored the partial specifier and sent the whole part, the quirk
|
|
612
|
+
// download() tolerates the same way: keep what was asked for
|
|
613
|
+
client.log.warn({
|
|
614
|
+
msg: 'Server returned more than the requested window, truncating the part',
|
|
615
|
+
part: key,
|
|
616
|
+
maxBytes,
|
|
617
|
+
received: content.length,
|
|
618
|
+
cid: client.id
|
|
619
|
+
});
|
|
620
|
+
content = content.subarray(0, maxBytes);
|
|
621
|
+
}
|
|
497
622
|
if (!data[key]) {
|
|
498
623
|
data[key] = { content };
|
|
499
624
|
}
|
|
@@ -19,6 +19,10 @@ const safeNumber = (value) => {
|
|
|
19
19
|
let num = Math.round(Number(value));
|
|
20
20
|
return Number.isSafeInteger(num) && num >= 0 ? num : 0;
|
|
21
21
|
};
|
|
22
|
+
// Log output stands in a placeholder for any value longer than this, so a log entry never
|
|
23
|
+
// carries a copy of a large token (a message body, a long unquoted server token)
|
|
24
|
+
const LOG_VALUE_LIMIT = 100;
|
|
25
|
+
const logPlaceholder = (length, kind) => `"(* ${length}B ${kind} *)"`;
|
|
22
26
|
// Characters that cannot appear in an IMAP quoted string: CR and LF terminate a
|
|
23
27
|
// command line, and NUL is outside the CHAR production entirely. A value carrying
|
|
24
28
|
// any of them has to be sent as a literal, so quoting it is never correct.
|
|
@@ -127,8 +131,8 @@ async function compiler(response, options) {
|
|
|
127
131
|
return;
|
|
128
132
|
}
|
|
129
133
|
if (typeof node === 'string' || Buffer.isBuffer(node)) {
|
|
130
|
-
if (isLogging && node.length >
|
|
131
|
-
resp.push(emitEntry(
|
|
134
|
+
if (isLogging && node.length > LOG_VALUE_LIMIT) {
|
|
135
|
+
resp.push(emitEntry(logPlaceholder(node.length, 'string')));
|
|
132
136
|
}
|
|
133
137
|
else {
|
|
134
138
|
resp.push(emitEntry(isLogging ? JSON.stringify(node.toString()) : quoteString(node.toString())));
|
|
@@ -147,7 +151,7 @@ async function compiler(response, options) {
|
|
|
147
151
|
switch (node.type.toUpperCase()) {
|
|
148
152
|
case 'LITERAL':
|
|
149
153
|
if (isLogging) {
|
|
150
|
-
resp.push(emitEntry(
|
|
154
|
+
resp.push(emitEntry(logPlaceholder(node.value.length, 'literal')));
|
|
151
155
|
}
|
|
152
156
|
else {
|
|
153
157
|
// The literal size marker counts octets - string values are written as
|
|
@@ -179,8 +183,8 @@ async function compiler(response, options) {
|
|
|
179
183
|
}
|
|
180
184
|
break;
|
|
181
185
|
case 'STRING':
|
|
182
|
-
if (isLogging && node.value.length >
|
|
183
|
-
resp.push(emitEntry(
|
|
186
|
+
if (isLogging && node.value.length > LOG_VALUE_LIMIT) {
|
|
187
|
+
resp.push(emitEntry(logPlaceholder(node.value.length, 'string')));
|
|
184
188
|
}
|
|
185
189
|
else {
|
|
186
190
|
val = (node.value || '').toString();
|
|
@@ -238,11 +242,16 @@ async function compiler(response, options) {
|
|
|
238
242
|
case 'SECTION':
|
|
239
243
|
val = (node.value || '').toString();
|
|
240
244
|
if (!node.section || val) {
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
if (node.value === '' ||
|
|
245
|
+
if (isLogging && val.length > LOG_VALUE_LIMIT) {
|
|
246
|
+
// A server can put a line's worth of bytes in one unquoted token
|
|
247
|
+
val = logPlaceholder(val.length, 'atom');
|
|
248
|
+
}
|
|
249
|
+
else if (node.value === '' ||
|
|
250
|
+
imapFormalSyntax.verify(val.charAt(0) === '\\' ? val.slice(1) : val, imapFormalSyntax['ATOM-CHAR']()) >= 0) {
|
|
251
|
+
// An empty value, or one with a character outside ATOM-CHAR (checked
|
|
252
|
+
// past a leading backslash, as system flags like \Seen carry one), goes
|
|
253
|
+
// out as an IMAP quoted string. JSON.stringify is used only for log
|
|
254
|
+
// output, where values are display-escaped.
|
|
246
255
|
val = isLogging ? JSON.stringify(val) : quoteString(val);
|
|
247
256
|
}
|
|
248
257
|
resp.push(emitEntry(val));
|
|
@@ -2,6 +2,17 @@ import type { ImapFlowError } from '../errors.js';
|
|
|
2
2
|
export declare const MAX_LITERAL_SIZE: number;
|
|
3
3
|
export declare const MAX_LINE_SIZE: number;
|
|
4
4
|
export declare const MAX_RESPONSE_SIZE: number;
|
|
5
|
+
export declare const ERROR_CONTEXT_LENGTH = 1024;
|
|
6
|
+
/**
|
|
7
|
+
* The input a parse error carries: a bounded prefix with the full length.
|
|
8
|
+
*
|
|
9
|
+
* @param input - The input that failed to parse
|
|
10
|
+
* @returns The bounded prefix and the full length
|
|
11
|
+
*/
|
|
12
|
+
export declare const boundedInput: (input: string) => {
|
|
13
|
+
input: string;
|
|
14
|
+
inputLength: number;
|
|
15
|
+
};
|
|
5
16
|
/**
|
|
6
17
|
* Normalizes a configured size limit. A non-negative integer is honored as-is (including 0, which
|
|
7
18
|
* means "reject anything non-empty"), and `Infinity` disables the limit; anything else falls back
|
|
@@ -17,6 +17,20 @@ export const MAX_LINE_SIZE = MAX_LITERAL_SIZE;
|
|
|
17
17
|
// of exactly the maximum permitted size impossible to receive. Configuring both limits calls for
|
|
18
18
|
// the same headroom - set maxResponseSize above maxLiteralSize, not equal to it.
|
|
19
19
|
export const MAX_RESPONSE_SIZE = 2 * MAX_LITERAL_SIZE;
|
|
20
|
+
// How much of the offending input a parse error carries along. The error travels whole into log
|
|
21
|
+
// entries (pino copies every property of a logged error) and into the rejection of the command
|
|
22
|
+
// the line belonged to, and the line can be as long as the configured line cap.
|
|
23
|
+
export const ERROR_CONTEXT_LENGTH = 1024;
|
|
24
|
+
/**
|
|
25
|
+
* The input a parse error carries: a bounded prefix with the full length.
|
|
26
|
+
*
|
|
27
|
+
* @param input - The input that failed to parse
|
|
28
|
+
* @returns The bounded prefix and the full length
|
|
29
|
+
*/
|
|
30
|
+
export const boundedInput = (input) => ({
|
|
31
|
+
input: input.slice(0, ERROR_CONTEXT_LENGTH),
|
|
32
|
+
inputLength: input.length
|
|
33
|
+
});
|
|
20
34
|
/**
|
|
21
35
|
* Normalizes a configured size limit. A non-negative integer is honored as-is (including 0, which
|
|
22
36
|
* means "reject anything non-empty"), and `Infinity` disables the limit; anything else falls back
|
|
@@ -21,6 +21,16 @@ export declare class ParserInstance {
|
|
|
21
21
|
* @param options.literals - Pre-parsed literal values from the stream.
|
|
22
22
|
*/
|
|
23
23
|
constructor(input?: Buffer | string | null | undefined, options?: ParserOptions | undefined);
|
|
24
|
+
/**
|
|
25
|
+
* The context a parse error carries: a bounded prefix of the input (and of the element that
|
|
26
|
+
* failed, when there is one) with their full lengths, and the position.
|
|
27
|
+
*
|
|
28
|
+
* @param element - The element that failed to parse
|
|
29
|
+
* @returns The context to attach to the error
|
|
30
|
+
*/
|
|
31
|
+
errorContext(element?: string | undefined): {
|
|
32
|
+
[key: string]: unknown;
|
|
33
|
+
};
|
|
24
34
|
/**
|
|
25
35
|
* Extracts and returns the IMAP tag from the beginning of the response.
|
|
26
36
|
* The tag is typically "*" for untagged responses, "+" for continuation requests,
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
/* eslint new-cap: 0 */
|
|
2
2
|
import imapFormalSyntax from './imap-formal-syntax.js';
|
|
3
3
|
import { TokenParser } from './token-parser.js';
|
|
4
|
+
import { boundedInput } from './limits.js';
|
|
4
5
|
/**
|
|
5
6
|
* Parses a single IMAP response line into its structural components: tag, command,
|
|
6
7
|
* and attributes. Handles status responses (OK, NO, BAD, PREAUTH, BYE) with their
|
|
@@ -21,6 +22,22 @@ export class ParserInstance {
|
|
|
21
22
|
this.remainder = this.input;
|
|
22
23
|
this.pos = 0;
|
|
23
24
|
}
|
|
25
|
+
/**
|
|
26
|
+
* The context a parse error carries: a bounded prefix of the input (and of the element that
|
|
27
|
+
* failed, when there is one) with their full lengths, and the position.
|
|
28
|
+
*
|
|
29
|
+
* @param element - The element that failed to parse
|
|
30
|
+
* @returns The context to attach to the error
|
|
31
|
+
*/
|
|
32
|
+
errorContext(element) {
|
|
33
|
+
let context = { ...boundedInput(this.input), pos: this.pos };
|
|
34
|
+
if (element !== undefined) {
|
|
35
|
+
let bounded = boundedInput(element);
|
|
36
|
+
context.element = bounded.input;
|
|
37
|
+
context.elementLength = bounded.inputLength;
|
|
38
|
+
}
|
|
39
|
+
return context;
|
|
40
|
+
}
|
|
24
41
|
/**
|
|
25
42
|
* Extracts and returns the IMAP tag from the beginning of the response.
|
|
26
43
|
* The tag is typically "*" for untagged responses, "+" for continuation requests,
|
|
@@ -122,7 +139,7 @@ export class ParserInstance {
|
|
|
122
139
|
if (/^\s/.test(this.remainder)) {
|
|
123
140
|
let error = new Error(`Unexpected whitespace at position ${this.pos} [E1]`);
|
|
124
141
|
error.code = 'ParserError1';
|
|
125
|
-
error.parserContext =
|
|
142
|
+
error.parserContext = this.errorContext();
|
|
126
143
|
throw error;
|
|
127
144
|
}
|
|
128
145
|
if ((match = this.remainder.match(/^\s*[^\s]+(?=\s|$)/))) {
|
|
@@ -136,9 +153,7 @@ export class ParserInstance {
|
|
|
136
153
|
let error = new Error(`Server returned an error: ${this.input}`);
|
|
137
154
|
error.code = 'ParserErrorExchange';
|
|
138
155
|
error.parserContext = {
|
|
139
|
-
|
|
140
|
-
element,
|
|
141
|
-
pos: this.pos,
|
|
156
|
+
...this.errorContext(element),
|
|
142
157
|
value: {
|
|
143
158
|
tag: '*',
|
|
144
159
|
command: 'BAD',
|
|
@@ -149,14 +164,14 @@ export class ParserInstance {
|
|
|
149
164
|
}
|
|
150
165
|
let error = new Error(`Unexpected char at position ${this.pos + errPos} [E2: ${JSON.stringify(element.charAt(errPos))}]`);
|
|
151
166
|
error.code = 'ParserError2';
|
|
152
|
-
error.parserContext =
|
|
167
|
+
error.parserContext = this.errorContext(element);
|
|
153
168
|
throw error;
|
|
154
169
|
}
|
|
155
170
|
}
|
|
156
171
|
else {
|
|
157
172
|
let error = new Error(`Unexpected end of input at position ${this.pos} [E3]`);
|
|
158
173
|
error.code = 'ParserError3';
|
|
159
|
-
error.parserContext =
|
|
174
|
+
error.parserContext = this.errorContext();
|
|
160
175
|
throw error;
|
|
161
176
|
}
|
|
162
177
|
this.pos += match[0].length;
|
|
@@ -177,13 +192,13 @@ export class ParserInstance {
|
|
|
177
192
|
}
|
|
178
193
|
let error = new Error(`Unexpected end of input at position ${this.pos} [E4]`);
|
|
179
194
|
error.code = 'ParserError4';
|
|
180
|
-
error.parserContext =
|
|
195
|
+
error.parserContext = this.errorContext();
|
|
181
196
|
throw error;
|
|
182
197
|
}
|
|
183
198
|
if (imapFormalSyntax.verify(this.remainder.charAt(0), imapFormalSyntax.SP()) >= 0) {
|
|
184
199
|
let error = new Error(`Unexpected char at position ${this.pos} [E5: ${JSON.stringify(this.remainder.charAt(0))}]`);
|
|
185
200
|
error.code = 'ParserError5';
|
|
186
|
-
error.parserContext =
|
|
201
|
+
error.parserContext = this.errorContext(this.remainder);
|
|
187
202
|
throw error;
|
|
188
203
|
}
|
|
189
204
|
this.pos++;
|
|
@@ -201,13 +216,13 @@ export class ParserInstance {
|
|
|
201
216
|
if (!this.remainder.length) {
|
|
202
217
|
let error = new Error(`Unexpected end of input at position ${this.pos} [E6]`);
|
|
203
218
|
error.code = 'ParserError6';
|
|
204
|
-
error.parserContext =
|
|
219
|
+
error.parserContext = this.errorContext();
|
|
205
220
|
throw error;
|
|
206
221
|
}
|
|
207
222
|
if (/^\s/.test(this.remainder)) {
|
|
208
223
|
let error = new Error(`Unexpected whitespace at position ${this.pos} [E7]`);
|
|
209
224
|
error.code = 'ParserError7';
|
|
210
|
-
error.parserContext =
|
|
225
|
+
error.parserContext = this.errorContext(this.remainder);
|
|
211
226
|
throw error;
|
|
212
227
|
}
|
|
213
228
|
const tokenParser = new TokenParser(this, this.pos, this.remainder, this.options);
|