@bobfrankston/mailx-imap 0.1.28 → 0.1.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +46 -14
- package/index.js +522 -190
- package/package.json +11 -11
package/index.d.ts
CHANGED
|
@@ -105,11 +105,18 @@ export declare class ImapManager extends EventEmitter {
|
|
|
105
105
|
searchAndFetchOnServer(accountId: string, folderId: number, mailboxPath: string, criteria: any): Promise<number[]>;
|
|
106
106
|
/** Create a fresh IMAP client for an account (public access for API endpoints) */
|
|
107
107
|
createPublicClient(accountId: string): Promise<any>;
|
|
108
|
-
/** Persistent operational
|
|
109
|
-
*
|
|
110
|
-
*
|
|
111
|
-
* in the queue lets interactive clicks jump ahead of background prefetch. */
|
|
108
|
+
/** Persistent slow-lane operational connection per account. Used by sync,
|
|
109
|
+
* prefetch, outbox-append, large backfills, and any other "this might
|
|
110
|
+
* take a while" operation. */
|
|
112
111
|
private opsClients;
|
|
112
|
+
/** Lazy-allocated fast-lane client per account (C123). Lives alongside
|
|
113
|
+
* `opsClients` so click-time body fetches and flag toggles don't have
|
|
114
|
+
* to wait behind a multi-minute prefetch / backfill on the slow client.
|
|
115
|
+
* Created on the first fast-lane `withConnection` call; reused while
|
|
116
|
+
* alive; recreated on stale-detect or error-discard like opsClients.
|
|
117
|
+
* Costs +1 IMAP socket per active account, well under any reasonable
|
|
118
|
+
* per-user-IP cap (Dovecot's default is 20). */
|
|
119
|
+
private fastClients;
|
|
113
120
|
/** Two-lane operation queue per account — interactive ops (body fetch on
|
|
114
121
|
* click, flag toggle) drain before background ops (sync, prefetch). FIFO
|
|
115
122
|
* within each lane. The single ops connection means there's never a race
|
|
@@ -122,8 +129,16 @@ export declare class ImapManager extends EventEmitter {
|
|
|
122
129
|
* case (e.g. bobma + bobma2 both on imap.iecc.com). */
|
|
123
130
|
private hostSemaphores;
|
|
124
131
|
private static readonly HOST_PERMITS;
|
|
125
|
-
/** Get (or create)
|
|
126
|
-
*
|
|
132
|
+
/** Get (or create) a persistent connection for an account on the named
|
|
133
|
+
* lane. Two lanes today: `slow` (the original `opsClients` map — sync,
|
|
134
|
+
* prefetch, backfill, outbox) and `fast` (the C123 `fastClients` map —
|
|
135
|
+
* click-time body fetch, flag toggles, anything user-driven). Same
|
|
136
|
+
* stale-detect + reconnect semantics on both. logout() is wrapped as a
|
|
137
|
+
* no-op so legacy callers don't close the persistent client. */
|
|
138
|
+
private getLaneClient;
|
|
139
|
+
/** Backwards-compat shim — callers outside the queue still ask for the
|
|
140
|
+
* ops client by historical name. New code should prefer withConnection
|
|
141
|
+
* with `slow:true` for slow operations. */
|
|
127
142
|
private getOpsClient;
|
|
128
143
|
/** Run an operation on the account's connection — queued, sequential, no concurrency */
|
|
129
144
|
/** Run an operation against the account's single ops connection. Tasks
|
|
@@ -143,9 +158,10 @@ export declare class ImapManager extends EventEmitter {
|
|
|
143
158
|
slow?: boolean;
|
|
144
159
|
timeoutMs?: number;
|
|
145
160
|
}): Promise<T>;
|
|
146
|
-
/** Run the next queued task. Fast
|
|
147
|
-
*
|
|
148
|
-
* flag prevents reentrant draining
|
|
161
|
+
/** Run the next queued task on each lane. Fast and slow lanes drain
|
|
162
|
+
* CONCURRENTLY — each on its own connection (C123). FIFO within each
|
|
163
|
+
* lane. The running flag per lane prevents reentrant draining of the
|
|
164
|
+
* same lane. */
|
|
149
165
|
private drainOpsQueue;
|
|
150
166
|
/** Acquire one slot of the per-host connection semaphore. Returns a release
|
|
151
167
|
* function — call exactly once when the socket is closed. Used by
|
|
@@ -161,12 +177,14 @@ export declare class ImapManager extends EventEmitter {
|
|
|
161
177
|
* `purpose` is a short tag printed alongside the `[conn+]` log so we can
|
|
162
178
|
* tell which code path (ops/idle/etc.) opened each connection. */
|
|
163
179
|
private newClient;
|
|
164
|
-
/** Force-close every IMAP socket for an account —
|
|
165
|
-
* ones in openClients (e.g. an IDLE
|
|
166
|
-
*
|
|
167
|
-
*
|
|
180
|
+
/** Force-close every IMAP socket for an account — both lane clients
|
|
181
|
+
* (ops + fast) plus any lingering ones in openClients (e.g. an IDLE
|
|
182
|
+
* watcher in flight). Used during account removal and disconnectOps
|
|
183
|
+
* so the server's connection slots free immediately rather than
|
|
184
|
+
* waiting for socket idle timeouts. */
|
|
168
185
|
closeAllClients(accountId: string): Promise<void>;
|
|
169
|
-
/** Disconnect the persistent operational
|
|
186
|
+
/** Disconnect the persistent operational connections (both lanes) for
|
|
187
|
+
* an account. */
|
|
170
188
|
disconnectOps(accountId: string): Promise<void>;
|
|
171
189
|
/** Legacy entry: returns the shared persistent ops client. Most callers
|
|
172
190
|
* should be using `withConnection()` instead — that gives proper
|
|
@@ -191,6 +209,20 @@ export declare class ImapManager extends EventEmitter {
|
|
|
191
209
|
syncFolders(accountId: string, client?: any): Promise<Folder[]>;
|
|
192
210
|
/** Store a batch of messages to DB immediately — used by onChunk for incremental sync */
|
|
193
211
|
private storeMessages;
|
|
212
|
+
/** Insert a local row for a message we just APPENDed (Sent / Drafts).
|
|
213
|
+
* Uses the source bytes we already have plus the UID returned by the
|
|
214
|
+
* server's APPENDUID response to build the local row directly — no
|
|
215
|
+
* IMAP SELECT/SEARCH/FETCH round-trip required. This is the user-
|
|
216
|
+
* visible fix for "I sent a message but it doesn't show up in mailx
|
|
217
|
+
* Sent until the broad sync finally completes" (which on slow servers
|
|
218
|
+
* can be minutes — and which Thunderbird sidesteps entirely because
|
|
219
|
+
* it inserts its own row at APPEND time too).
|
|
220
|
+
*
|
|
221
|
+
* Fires the same emits as a normal sync so the UI updates. */
|
|
222
|
+
insertLocalRowFromSource(accountId: string, folder: {
|
|
223
|
+
id: number;
|
|
224
|
+
path: string;
|
|
225
|
+
}, uid: number, source: string, flags: string[]): Promise<void>;
|
|
194
226
|
/** Sync messages for a specific folder */
|
|
195
227
|
syncFolder(accountId: string, folderId: number, client?: any): Promise<number>;
|
|
196
228
|
/** Sync all folders for all accounts */
|
package/index.js
CHANGED
|
@@ -327,11 +327,18 @@ export class ImapManager extends EventEmitter {
|
|
|
327
327
|
// All operations on an account are serialized through an operation queue.
|
|
328
328
|
// No semaphore, no pool, no per-operation connect/disconnect.
|
|
329
329
|
// IDLE uses a separate connection (see startWatching).
|
|
330
|
-
/** Persistent operational
|
|
331
|
-
*
|
|
332
|
-
*
|
|
333
|
-
* in the queue lets interactive clicks jump ahead of background prefetch. */
|
|
330
|
+
/** Persistent slow-lane operational connection per account. Used by sync,
|
|
331
|
+
* prefetch, outbox-append, large backfills, and any other "this might
|
|
332
|
+
* take a while" operation. */
|
|
334
333
|
opsClients = new Map();
|
|
334
|
+
/** Lazy-allocated fast-lane client per account (C123). Lives alongside
|
|
335
|
+
* `opsClients` so click-time body fetches and flag toggles don't have
|
|
336
|
+
* to wait behind a multi-minute prefetch / backfill on the slow client.
|
|
337
|
+
* Created on the first fast-lane `withConnection` call; reused while
|
|
338
|
+
* alive; recreated on stale-detect or error-discard like opsClients.
|
|
339
|
+
* Costs +1 IMAP socket per active account, well under any reasonable
|
|
340
|
+
* per-user-IP cap (Dovecot's default is 20). */
|
|
341
|
+
fastClients = new Map();
|
|
335
342
|
/** Two-lane operation queue per account — interactive ops (body fetch on
|
|
336
343
|
* click, flag toggle) drain before background ops (sync, prefetch). FIFO
|
|
337
344
|
* within each lane. The single ops connection means there's never a race
|
|
@@ -344,10 +351,15 @@ export class ImapManager extends EventEmitter {
|
|
|
344
351
|
* case (e.g. bobma + bobma2 both on imap.iecc.com). */
|
|
345
352
|
hostSemaphores = new Map();
|
|
346
353
|
static HOST_PERMITS = 4;
|
|
347
|
-
/** Get (or create)
|
|
348
|
-
*
|
|
349
|
-
|
|
350
|
-
|
|
354
|
+
/** Get (or create) a persistent connection for an account on the named
|
|
355
|
+
* lane. Two lanes today: `slow` (the original `opsClients` map — sync,
|
|
356
|
+
* prefetch, backfill, outbox) and `fast` (the C123 `fastClients` map —
|
|
357
|
+
* click-time body fetch, flag toggles, anything user-driven). Same
|
|
358
|
+
* stale-detect + reconnect semantics on both. logout() is wrapped as a
|
|
359
|
+
* no-op so legacy callers don't close the persistent client. */
|
|
360
|
+
async getLaneClient(accountId, lane) {
|
|
361
|
+
const map = lane === "fast" ? this.fastClients : this.opsClients;
|
|
362
|
+
let client = map.get(accountId);
|
|
351
363
|
if (client) {
|
|
352
364
|
// C38: health-check the cached client before returning. If the
|
|
353
365
|
// underlying socket is dead (Dovecot silently dropped IDLE after
|
|
@@ -363,19 +375,25 @@ export class ImapManager extends EventEmitter {
|
|
|
363
375
|
await (client._realLogout || client.logout)();
|
|
364
376
|
}
|
|
365
377
|
catch { /* */ }
|
|
366
|
-
|
|
367
|
-
console.log(` [conn] ${accountId}: stale
|
|
378
|
+
map.delete(accountId);
|
|
379
|
+
console.log(` [conn] ${accountId}: stale ${lane} client detected — reconnecting`);
|
|
368
380
|
client = undefined;
|
|
369
381
|
}
|
|
370
|
-
client = await this.newClient(accountId, "ops");
|
|
382
|
+
client = await this.newClient(accountId, lane === "fast" ? "fast" : "ops");
|
|
371
383
|
// Wrap logout as no-op — this is a persistent connection. The
|
|
372
384
|
// newClient wrapper's close-counter runs on `_realLogout`.
|
|
373
385
|
const realLogout = client.logout.bind(client);
|
|
374
386
|
client.logout = async () => { };
|
|
375
387
|
client._realLogout = realLogout;
|
|
376
|
-
|
|
388
|
+
map.set(accountId, client);
|
|
377
389
|
return client;
|
|
378
390
|
}
|
|
391
|
+
/** Backwards-compat shim — callers outside the queue still ask for the
|
|
392
|
+
* ops client by historical name. New code should prefer withConnection
|
|
393
|
+
* with `slow:true` for slow operations. */
|
|
394
|
+
async getOpsClient(accountId) {
|
|
395
|
+
return this.getLaneClient(accountId, "slow");
|
|
396
|
+
}
|
|
379
397
|
/** Run an operation on the account's connection — queued, sequential, no concurrency */
|
|
380
398
|
/** Run an operation against the account's single ops connection. Tasks
|
|
381
399
|
* queue strictly sequentially per account — only one IMAP command in
|
|
@@ -393,12 +411,13 @@ export class ImapManager extends EventEmitter {
|
|
|
393
411
|
async withConnection(accountId, fn, opts = {}) {
|
|
394
412
|
let queue = this.opsQueues.get(accountId);
|
|
395
413
|
if (!queue) {
|
|
396
|
-
queue = { fast: [], slow: [],
|
|
414
|
+
queue = { fast: [], slow: [], runningFast: false, runningSlow: false };
|
|
397
415
|
this.opsQueues.set(accountId, queue);
|
|
398
416
|
}
|
|
417
|
+
const lane = opts.slow ? "slow" : "fast";
|
|
399
418
|
// Per-task wall-clock cap. Without one, a wedged IMAP command (TCP
|
|
400
419
|
// half-open, server stalled mid-FETCH) keeps the queue's running flag
|
|
401
|
-
// set forever and every subsequent
|
|
420
|
+
// set forever and every subsequent same-lane task — including the
|
|
402
421
|
// retry button the user just hit — waits behind it. Default is
|
|
403
422
|
// generous; callers driving user-visible reads pass a tighter value.
|
|
404
423
|
const timeoutMs = opts.timeoutMs ?? 90_000;
|
|
@@ -406,11 +425,11 @@ export class ImapManager extends EventEmitter {
|
|
|
406
425
|
const task = async () => {
|
|
407
426
|
let timer;
|
|
408
427
|
try {
|
|
409
|
-
const client = await this.
|
|
428
|
+
const client = await this.getLaneClient(accountId, lane);
|
|
410
429
|
const result = await Promise.race([
|
|
411
430
|
fn(client),
|
|
412
431
|
new Promise((_, rej) => {
|
|
413
|
-
timer = setTimeout(() => rej(new Error(
|
|
432
|
+
timer = setTimeout(() => rej(new Error(`${lane}-ops timeout after ${Math.round(timeoutMs / 1000)}s — discarding client`)), timeoutMs);
|
|
414
433
|
}),
|
|
415
434
|
]);
|
|
416
435
|
clearTimeout(timer);
|
|
@@ -418,12 +437,15 @@ export class ImapManager extends EventEmitter {
|
|
|
418
437
|
}
|
|
419
438
|
catch (e) {
|
|
420
439
|
clearTimeout(timer);
|
|
421
|
-
// Discard client on any error —
|
|
422
|
-
// socket
|
|
423
|
-
//
|
|
424
|
-
//
|
|
425
|
-
|
|
426
|
-
|
|
440
|
+
// Discard the lane's client on any error — a half-broken
|
|
441
|
+
// socket poisons every subsequent request on that lane.
|
|
442
|
+
// Don't touch the OTHER lane's client; the lanes are
|
|
443
|
+
// independent. Destroy synchronously kills the in-flight
|
|
444
|
+
// command's socket so the underlying promise rejects and
|
|
445
|
+
// stops holding state.
|
|
446
|
+
const map = lane === "fast" ? this.fastClients : this.opsClients;
|
|
447
|
+
const stale = map.get(accountId);
|
|
448
|
+
map.delete(accountId);
|
|
427
449
|
if (stale) {
|
|
428
450
|
try {
|
|
429
451
|
await (stale._realLogout || stale.logout)();
|
|
@@ -437,25 +459,33 @@ export class ImapManager extends EventEmitter {
|
|
|
437
459
|
reject(e);
|
|
438
460
|
}
|
|
439
461
|
};
|
|
440
|
-
|
|
462
|
+
queue[lane].push(task);
|
|
441
463
|
this.drainOpsQueue(accountId);
|
|
442
464
|
});
|
|
443
465
|
}
|
|
444
|
-
/** Run the next queued task. Fast
|
|
445
|
-
*
|
|
446
|
-
* flag prevents reentrant draining
|
|
466
|
+
/** Run the next queued task on each lane. Fast and slow lanes drain
|
|
467
|
+
* CONCURRENTLY — each on its own connection (C123). FIFO within each
|
|
468
|
+
* lane. The running flag per lane prevents reentrant draining of the
|
|
469
|
+
* same lane. */
|
|
447
470
|
drainOpsQueue(accountId) {
|
|
448
471
|
const queue = this.opsQueues.get(accountId);
|
|
449
|
-
if (!queue
|
|
472
|
+
if (!queue)
|
|
450
473
|
return;
|
|
451
|
-
const
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
474
|
+
const drainLane = (lane) => {
|
|
475
|
+
const runningKey = lane === "fast" ? "runningFast" : "runningSlow";
|
|
476
|
+
if (queue[runningKey])
|
|
477
|
+
return;
|
|
478
|
+
const next = queue[lane].shift();
|
|
479
|
+
if (!next)
|
|
480
|
+
return;
|
|
481
|
+
queue[runningKey] = true;
|
|
482
|
+
next().finally(() => {
|
|
483
|
+
queue[runningKey] = false;
|
|
484
|
+
drainLane(lane);
|
|
485
|
+
});
|
|
486
|
+
};
|
|
487
|
+
drainLane("fast");
|
|
488
|
+
drainLane("slow");
|
|
459
489
|
}
|
|
460
490
|
/** Acquire one slot of the per-host connection semaphore. Returns a release
|
|
461
491
|
* function — call exactly once when the socket is closed. Used by
|
|
@@ -506,7 +536,15 @@ export class ImapManager extends EventEmitter {
|
|
|
506
536
|
const releaseHostSlot = await this.acquireHostSlot(host);
|
|
507
537
|
let client;
|
|
508
538
|
try {
|
|
509
|
-
|
|
539
|
+
// Verbose IMAP wire trace for ops connections only — that's the
|
|
540
|
+
// lane where commands have been hanging silently with no
|
|
541
|
+
// heartbeat / wall-clock fire / reject. Need to see the actual
|
|
542
|
+
// commands sent and bytes received to pinpoint where the
|
|
543
|
+
// pendingCommand is getting lost. Fast lane (C123) shares the
|
|
544
|
+
// verbose treatment so click-time wedges show too. Other lanes
|
|
545
|
+
// (idle, quickCheck) stay quiet so the log doesn't drown.
|
|
546
|
+
const cfgWithVerbose = (purpose === "ops" || purpose === "fast") ? { ...config, verbose: true } : config;
|
|
547
|
+
client = new CompatImapClient(cfgWithVerbose, this.transportFactory);
|
|
510
548
|
}
|
|
511
549
|
catch (e) {
|
|
512
550
|
releaseHostSlot();
|
|
@@ -552,22 +590,25 @@ export class ImapManager extends EventEmitter {
|
|
|
552
590
|
}
|
|
553
591
|
return client;
|
|
554
592
|
}
|
|
555
|
-
/** Force-close every IMAP socket for an account —
|
|
556
|
-
* ones in openClients (e.g. an IDLE
|
|
557
|
-
*
|
|
558
|
-
*
|
|
593
|
+
/** Force-close every IMAP socket for an account — both lane clients
|
|
594
|
+
* (ops + fast) plus any lingering ones in openClients (e.g. an IDLE
|
|
595
|
+
* watcher in flight). Used during account removal and disconnectOps
|
|
596
|
+
* so the server's connection slots free immediately rather than
|
|
597
|
+
* waiting for socket idle timeouts. */
|
|
559
598
|
async closeAllClients(accountId) {
|
|
560
|
-
const
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
599
|
+
for (const map of [this.opsClients, this.fastClients]) {
|
|
600
|
+
const c = map.get(accountId);
|
|
601
|
+
map.delete(accountId);
|
|
602
|
+
if (c) {
|
|
603
|
+
try {
|
|
604
|
+
await (c._realLogout || c.logout)();
|
|
605
|
+
}
|
|
606
|
+
catch { /* */ }
|
|
607
|
+
try {
|
|
608
|
+
c.destroy?.();
|
|
609
|
+
}
|
|
610
|
+
catch { /* */ }
|
|
569
611
|
}
|
|
570
|
-
catch { /* */ }
|
|
571
612
|
}
|
|
572
613
|
const open = this.openClients.get(accountId);
|
|
573
614
|
if (open) {
|
|
@@ -584,23 +625,26 @@ export class ImapManager extends EventEmitter {
|
|
|
584
625
|
open.clear();
|
|
585
626
|
}
|
|
586
627
|
}
|
|
587
|
-
/** Disconnect the persistent operational
|
|
628
|
+
/** Disconnect the persistent operational connections (both lanes) for
|
|
629
|
+
* an account. */
|
|
588
630
|
async disconnectOps(accountId) {
|
|
589
|
-
const
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
631
|
+
for (const [name, map] of [["ops", this.opsClients], ["fast", this.fastClients]]) {
|
|
632
|
+
const client = map.get(accountId);
|
|
633
|
+
map.delete(accountId);
|
|
634
|
+
if (client) {
|
|
635
|
+
// Force-close: don't wait for LOGOUT on a possibly dead socket
|
|
636
|
+
try {
|
|
637
|
+
const timeout = new Promise(r => setTimeout(r, 2000));
|
|
638
|
+
await Promise.race([(client._realLogout || client.logout)(), timeout]);
|
|
639
|
+
}
|
|
640
|
+
catch { /* */ }
|
|
641
|
+
// Destroy underlying socket if still open
|
|
642
|
+
try {
|
|
643
|
+
client.destroy?.();
|
|
644
|
+
}
|
|
645
|
+
catch { /* */ }
|
|
646
|
+
console.log(` [conn] ${accountId} (${name}): disconnected`);
|
|
601
647
|
}
|
|
602
|
-
catch { /* */ }
|
|
603
|
-
console.log(` [conn] ${accountId}: disconnected`);
|
|
604
648
|
}
|
|
605
649
|
}
|
|
606
650
|
/** Legacy entry: returns the shared persistent ops client. Most callers
|
|
@@ -884,6 +928,49 @@ export class ImapManager extends EventEmitter {
|
|
|
884
928
|
}
|
|
885
929
|
return stored;
|
|
886
930
|
}
|
|
931
|
+
/** Insert a local row for a message we just APPENDed (Sent / Drafts).
|
|
932
|
+
* Uses the source bytes we already have plus the UID returned by the
|
|
933
|
+
* server's APPENDUID response to build the local row directly — no
|
|
934
|
+
* IMAP SELECT/SEARCH/FETCH round-trip required. This is the user-
|
|
935
|
+
* visible fix for "I sent a message but it doesn't show up in mailx
|
|
936
|
+
* Sent until the broad sync finally completes" (which on slow servers
|
|
937
|
+
* can be minutes — and which Thunderbird sidesteps entirely because
|
|
938
|
+
* it inserts its own row at APPEND time too).
|
|
939
|
+
*
|
|
940
|
+
* Fires the same emits as a normal sync so the UI updates. */
|
|
941
|
+
async insertLocalRowFromSource(accountId, folder, uid, source, flags) {
|
|
942
|
+
const { simpleParser } = await import("mailparser");
|
|
943
|
+
const parsed = await simpleParser(source);
|
|
944
|
+
// Coerce mailparser AddressObject(s) into the flat `{name, address}[]`
|
|
945
|
+
// shape storeMessages's downstream toEmailAddresses expects.
|
|
946
|
+
const flat = (a) => {
|
|
947
|
+
if (!a)
|
|
948
|
+
return [];
|
|
949
|
+
if (Array.isArray(a))
|
|
950
|
+
return a.flatMap(x => x?.value || []);
|
|
951
|
+
return a.value || [];
|
|
952
|
+
};
|
|
953
|
+
const msg = {
|
|
954
|
+
uid,
|
|
955
|
+
messageId: parsed.messageId || "",
|
|
956
|
+
inReplyTo: parsed.inReplyTo || "",
|
|
957
|
+
date: parsed.date || new Date(),
|
|
958
|
+
subject: parsed.subject || "",
|
|
959
|
+
from: flat(parsed.from),
|
|
960
|
+
to: flat(parsed.to),
|
|
961
|
+
cc: flat(parsed.cc),
|
|
962
|
+
seen: flags.includes("\\Seen"),
|
|
963
|
+
flagged: flags.includes("\\Flagged"),
|
|
964
|
+
answered: flags.includes("\\Answered"),
|
|
965
|
+
draft: flags.includes("\\Draft"),
|
|
966
|
+
source,
|
|
967
|
+
size: source.length,
|
|
968
|
+
};
|
|
969
|
+
await this.storeMessages(accountId, folder.id, folder, [msg], 0);
|
|
970
|
+
this.db.recalcFolderCounts(folder.id);
|
|
971
|
+
this.emit("folderCountsChanged", accountId, {});
|
|
972
|
+
console.log(` [local-insert] ${folder.path} UID ${uid}: ${parsed.subject || "(no subject)"} (no IMAP roundtrip)`);
|
|
973
|
+
}
|
|
887
974
|
/** Sync messages for a specific folder */
|
|
888
975
|
async syncFolder(accountId, folderId, client) {
|
|
889
976
|
if (!client)
|
|
@@ -894,17 +981,72 @@ export class ImapManager extends EventEmitter {
|
|
|
894
981
|
if (!folder)
|
|
895
982
|
throw new Error(`Folder ${folderId} not found`);
|
|
896
983
|
this.emit("syncProgress", accountId, `sync:${folder.path}`, 0);
|
|
984
|
+
// Top-of-syncFolder breadcrumb — fires BEFORE any async I/O so a
|
|
985
|
+
// hang in STATUS or SELECT can be located by its absence of a
|
|
986
|
+
// following completion log. Without this, "Sent never showed up
|
|
987
|
+
// in the log" was indistinguishable from "Sent hadn't started"
|
|
988
|
+
// versus "Sent's STATUS call is hung."
|
|
989
|
+
const __sfStart = Date.now();
|
|
990
|
+
console.log(` [sync-enter] ${accountId}/${folder.path} (folderId=${folderId})`);
|
|
897
991
|
// Get the highest UID we already have for this folder. IMAP UIDs are
|
|
898
992
|
// monotonically increasing within a UIDVALIDITY (RFC 3501); a
|
|
899
993
|
// high-water mark is the right anchor for incremental fetch.
|
|
900
994
|
const highestUid = this.db.getHighestUid(accountId, folderId);
|
|
901
|
-
|
|
902
|
-
//
|
|
903
|
-
//
|
|
904
|
-
//
|
|
905
|
-
//
|
|
995
|
+
const localCount = this.db.getMessageCount(accountId, folderId);
|
|
996
|
+
// STATUS-before-SELECT — fast O(1) command that returns UIDNEXT and
|
|
997
|
+
// MESSAGES without touching the mailbox SELECT state. RFC 3501 §6.3.10.
|
|
998
|
+
// This is the difference between "INBOX works, Sent doesn't" on this
|
|
999
|
+
// server: INBOX's quickImapCheck uses STATUS first; non-INBOX folders
|
|
1000
|
+
// were going straight to SELECT every cycle, and SELECT on Sent has
|
|
1001
|
+
// been observed wedging this server's ops connection for minutes
|
|
1002
|
+
// (Thunderbird, which uses STATUS-before-SELECT, doesn't see this).
|
|
1003
|
+
// When the server's UIDNEXT-1 ≤ our highestUid AND MESSAGES count
|
|
1004
|
+
// matches our local count, NOTHING changed — skip SELECT entirely.
|
|
1005
|
+
// For first sync (highestUid = 0) skip the STATUS check and let the
|
|
1006
|
+
// existing first-sync path (sequence-FETCH the latest N) run.
|
|
1007
|
+
if (highestUid > 0) {
|
|
1008
|
+
try {
|
|
1009
|
+
console.log(` [sync-status] ${accountId}/${folder.path}: calling STATUS...`);
|
|
1010
|
+
const __statusT0 = Date.now();
|
|
1011
|
+
const status = await client.native?.getStatus?.(folder.path)
|
|
1012
|
+
?? await client.getStatus?.(folder.path);
|
|
1013
|
+
console.log(` [sync-status] ${accountId}/${folder.path}: STATUS returned in ${Date.now() - __statusT0}ms`);
|
|
1014
|
+
if (status && typeof status.uidNext === "number") {
|
|
1015
|
+
const serverHighest = status.uidNext - 1;
|
|
1016
|
+
const noNewUids = serverHighest <= highestUid;
|
|
1017
|
+
const countsMatch = typeof status.messages === "number" && status.messages === localCount;
|
|
1018
|
+
if (noNewUids && countsMatch) {
|
|
1019
|
+
// Nothing new server-side AND counts match — full
|
|
1020
|
+
// sync would be a no-op. Skip SELECT/SEARCH/FETCH
|
|
1021
|
+
// entirely. This is the path Sent takes most of the
|
|
1022
|
+
// time after the optimistic-APPEND-insert (1.0.573)
|
|
1023
|
+
// already filed the just-sent row locally.
|
|
1024
|
+
console.log(` [sync] ${accountId}/${folder.path}: STATUS clean (uidNext=${status.uidNext} messages=${status.messages}) — skipping SELECT`);
|
|
1025
|
+
this.emit("syncProgress", accountId, `sync:${folder.path}`, 100);
|
|
1026
|
+
return 0;
|
|
1027
|
+
}
|
|
1028
|
+
// Counts differ or server has new UIDs — fall through to
|
|
1029
|
+
// the full sync path below, which knows how to reconcile.
|
|
1030
|
+
console.log(` [sync] ${accountId}/${folder.path}: STATUS uidNext=${status.uidNext} messages=${status.messages} local=${localCount}/${highestUid} — proceeding to SELECT`);
|
|
1031
|
+
}
|
|
1032
|
+
}
|
|
1033
|
+
catch (statusErr) {
|
|
1034
|
+
// STATUS failed — fall through to SELECT (some servers don't
|
|
1035
|
+
// support STATUS on the currently-SELECTed mailbox; per
|
|
1036
|
+
// RFC 3501 §6.3.10 it's actually allowed everywhere but a
|
|
1037
|
+
// misbehaving server might refuse). The fallback is the
|
|
1038
|
+
// pre-1.0.574 behavior, so worst case we lose the speedup.
|
|
1039
|
+
console.log(` [sync] ${accountId}/${folder.path}: STATUS failed (${statusErr?.message || statusErr}) — falling back to SELECT`);
|
|
1040
|
+
}
|
|
1041
|
+
}
|
|
906
1042
|
console.log(` [sync] ${accountId}/${folder.path}: highestUid=${highestUid}, fetching...`);
|
|
907
1043
|
let messages;
|
|
1044
|
+
// Cache the server-UID list across set-diff and deletion-recon so we
|
|
1045
|
+
// don't pay for two `UID SEARCH` round-trips (the second one was
|
|
1046
|
+
// `UID SEARCH ALL` which on a 134k-message INBOX hangs for minutes
|
|
1047
|
+
// on this server). Set-diff populates this; deletion-recon reuses.
|
|
1048
|
+
let serverUidsCached = null;
|
|
1049
|
+
let serverUidsAreDateBounded = false;
|
|
908
1050
|
const firstSync = highestUid === 0;
|
|
909
1051
|
const historyDays = getHistoryDays(accountId);
|
|
910
1052
|
// historyDays=0 means "all". On first sync we still cap at 30 days
|
|
@@ -922,60 +1064,105 @@ export class ImapManager extends EventEmitter {
|
|
|
922
1064
|
const fetched = await client.fetchMessagesSinceUid(folder.path, highestUid, { source: false });
|
|
923
1065
|
// Filter out the last known message (IMAP * always returns at least one)
|
|
924
1066
|
messages = fetched.filter((m) => m.uid > highestUid);
|
|
925
|
-
//
|
|
926
|
-
//
|
|
1067
|
+
// Reconciliation by SET DIFFERENCE, not high-water-mark.
|
|
1068
|
+
//
|
|
1069
|
+
// The high-water-mark approach (fetch UIDs > highestUid) only
|
|
1070
|
+
// catches NEW arrivals. It cannot see anything below the local
|
|
1071
|
+
// lowest UID — so messages that exist on the server but predate
|
|
1072
|
+
// the user's first sync (or any other gap) are invisible forever.
|
|
1073
|
+
// The previous gap-fill clamped reconciliation to [lowestUid,
|
|
1074
|
+
// highestUid] which made that gap permanent.
|
|
1075
|
+
//
|
|
1076
|
+
// The right model: server UID set is the truth. Anything on
|
|
1077
|
+
// the server that's not local → fetch. Anything local that's
|
|
1078
|
+
// not on the server → reconcile-delete (handled separately by
|
|
1079
|
+
// the deferred-delete path, with grace + 50% safeguards).
|
|
1080
|
+
//
|
|
1081
|
+
// Bounded by a 5000-cap so a misbehaving server response or
|
|
1082
|
+
// brand-new account doesn't try to pull tens of thousands of
|
|
1083
|
+
// historical messages in one cycle. Hit the cap and the next
|
|
1084
|
+
// sync picks up the rest.
|
|
927
1085
|
const existingUids = this.db.getUidsForFolder(accountId, folderId);
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
1086
|
+
try {
|
|
1087
|
+
// Date-bound the server UID query when we have a history
|
|
1088
|
+
// window. UID SEARCH ALL on a 134k-message INBOX returns
|
|
1089
|
+
// 134k integers and triggers a heavyweight full-folder
|
|
1090
|
+
// scan on the server; UID SEARCH SINCE <date> returns only
|
|
1091
|
+
// UIDs in the window we actually care about. For
|
|
1092
|
+
// historyDays=0 ("keep everything") we still bound to the
|
|
1093
|
+
// last 5 years on incremental syncs — enough that we never
|
|
1094
|
+
// miss an in-flight message, without enumerating decades of
|
|
1095
|
+
// archive every cycle. First sync gets all UIDs (no anchor
|
|
1096
|
+
// yet so we have to compare against the empty local set
|
|
1097
|
+
// anyway).
|
|
1098
|
+
const SINCE_DAYS = effectiveDays > 0 ? effectiveDays : 1825;
|
|
1099
|
+
const sinceDate = new Date(Date.now() - SINCE_DAYS * 86400000);
|
|
1100
|
+
console.log(` [sync-uids] ${accountId}/${folder.path}: getUidsSince ${sinceDate.toISOString().slice(0, 10)}...`);
|
|
1101
|
+
const __uidsT0 = Date.now();
|
|
1102
|
+
const allServerUids = typeof client.getUidsSince === "function"
|
|
1103
|
+
? await client.getUidsSince(folder.path, sinceDate)
|
|
1104
|
+
: await client.getUids(folder.path);
|
|
1105
|
+
console.log(` [sync-uids] ${accountId}/${folder.path}: ${allServerUids.length} UIDs in ${Date.now() - __uidsT0}ms`);
|
|
1106
|
+
// Stash for the deletion-reconciliation block below — we
|
|
1107
|
+
// already have the date-bounded server UID list, no point
|
|
1108
|
+
// hitting the server a second time with UID SEARCH ALL.
|
|
1109
|
+
serverUidsCached = allServerUids;
|
|
1110
|
+
serverUidsAreDateBounded = true;
|
|
1111
|
+
const existingSet = new Set(existingUids);
|
|
1112
|
+
const newSet = new Set(messages.map(m => m.uid));
|
|
1113
|
+
const missingUids = allServerUids.filter((uid) => !existingSet.has(uid) && !newSet.has(uid));
|
|
1114
|
+
// Backfill chunk size. Use the passed-in `client` directly
|
|
1115
|
+
// (NOT a nested withConnection) — syncFolder is now wrapped
|
|
1116
|
+
// in withConnection at the call site, so the slow lane is
|
|
1117
|
+
// already locked for our duration. A nested withConnection
|
|
1118
|
+
// would deadlock waiting for the slot we hold.
|
|
1119
|
+
const BACKFILL_CHUNK_SIZE = 100;
|
|
1120
|
+
if (missingUids.length > 0 && missingUids.length <= 5000) {
|
|
1121
|
+
let minU = existingUids[0] ?? 0;
|
|
1122
|
+
for (let i = 1; i < existingUids.length; i++)
|
|
1123
|
+
if (existingUids[i] < minU)
|
|
1124
|
+
minU = existingUids[i];
|
|
1125
|
+
console.log(` ${folder.path}: ${missingUids.length} server-only UIDs (local lowest=${minU}, highest=${highestUid}) — fetching`);
|
|
1126
|
+
let recoveredTotal = 0;
|
|
1127
|
+
for (let i = 0; i < missingUids.length; i += BACKFILL_CHUNK_SIZE) {
|
|
1128
|
+
const chunk = missingUids.slice(i, i + BACKFILL_CHUNK_SIZE);
|
|
1129
|
+
const range = chunk.join(",");
|
|
1130
|
+
const recovered = await client.fetchMessages(folder.path, range, { source: false });
|
|
1131
|
+
messages.push(...recovered);
|
|
1132
|
+
recoveredTotal += recovered.length;
|
|
1133
|
+
console.log(` ${folder.path}: fetch ${recoveredTotal}/${missingUids.length}`);
|
|
952
1134
|
}
|
|
953
1135
|
}
|
|
954
|
-
|
|
955
|
-
console.
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
const backfill = await client.fetchMessageByDate(folder.path, chunkStart, new Date(oldestDate), { source: false });
|
|
969
|
-
const newBackfill = backfill.filter((m) => !existingUids.has(m.uid));
|
|
970
|
-
if (newBackfill.length > 0) {
|
|
971
|
-
console.log(` ${folder.path}: backfilling ${newBackfill.length} older messages (${chunkStart.toISOString().slice(0, 10)} → ${new Date(oldestDate).toISOString().slice(0, 10)})`);
|
|
972
|
-
messages.push(...newBackfill);
|
|
1136
|
+
else if (missingUids.length > 5000) {
|
|
1137
|
+
console.log(` ${folder.path}: ${missingUids.length} server-only UIDs — capped; will resume next cycle`);
|
|
1138
|
+
// Vanilla IMAP under stable UIDVALIDITY: higher UID =
|
|
1139
|
+
// later assignment ≈ more recent message (Dovecot/
|
|
1140
|
+
// Cyrus). Gmail-API path is separate (no temporal
|
|
1141
|
+
// meaning to its hash-UIDs).
|
|
1142
|
+
const cappedSlice = missingUids.sort((a, b) => b - a).slice(0, 5000);
|
|
1143
|
+
let recoveredTotal = 0;
|
|
1144
|
+
for (let i = 0; i < cappedSlice.length; i += BACKFILL_CHUNK_SIZE) {
|
|
1145
|
+
const chunk = cappedSlice.slice(i, i + BACKFILL_CHUNK_SIZE);
|
|
1146
|
+
const recovered = await client.fetchMessages(folder.path, chunk.join(","), { source: false });
|
|
1147
|
+
messages.push(...recovered);
|
|
1148
|
+
recoveredTotal += recovered.length;
|
|
1149
|
+
console.log(` ${folder.path}: fetch ${recoveredTotal}/5000 (capped)`);
|
|
973
1150
|
}
|
|
974
1151
|
}
|
|
975
|
-
catch (e) {
|
|
976
|
-
console.error(` ${folder.path}: backfill failed: ${e.message}`);
|
|
977
|
-
}
|
|
978
1152
|
}
|
|
1153
|
+
catch (e) {
|
|
1154
|
+
console.error(` ${folder.path}: reconciliation failed: ${e.message}`);
|
|
1155
|
+
}
|
|
1156
|
+
// Date-based backfill via SEARCH SINCE was here. Removed —
|
|
1157
|
+
// set-diff above already covers the same case (any server UID
|
|
1158
|
+
// not in our local set gets fetched, regardless of whether it's
|
|
1159
|
+
// above or below our local highest/lowest). SEARCH SINCE on a
|
|
1160
|
+
// large Dovecot folder walks INTERNALDATE on every message and
|
|
1161
|
+
// can take many minutes, which was wedging the whole syncFolder
|
|
1162
|
+
// call AFTER set-diff had succeeded — preventing syncAccount
|
|
1163
|
+
// from ever moving past INBOX to Sent / other folders. Don't
|
|
1164
|
+
// need both, and date-based-with-high-water-mark is exactly
|
|
1165
|
+
// the brittle model the set-diff replaces.
|
|
979
1166
|
}
|
|
980
1167
|
else {
|
|
981
1168
|
// First sync: fetch in chunks, store each chunk immediately for instant UI
|
|
@@ -1094,6 +1281,15 @@ export class ImapManager extends EventEmitter {
|
|
|
1094
1281
|
[folderId]: { total: folderInfo?.totalCount || 0, unread: folderInfo?.unreadCount || 0 }
|
|
1095
1282
|
});
|
|
1096
1283
|
}
|
|
1284
|
+
// Yield to the event loop between batches so a pending IPC
|
|
1285
|
+
// (e.g. user clicked a message) gets dispatched. Without this
|
|
1286
|
+
// yield, a 5000-message store loop blocked Node's event loop
|
|
1287
|
+
// for ~5s of synchronous SQLite work — clicks fired during
|
|
1288
|
+
// that window felt frozen because the getMessage IPC sat in
|
|
1289
|
+
// stdin until the loop finished. setImmediate runs after I/O
|
|
1290
|
+
// callbacks, which means stdin readable events fire first and
|
|
1291
|
+
// get serviced.
|
|
1292
|
+
await new Promise(r => setImmediate(r));
|
|
1097
1293
|
}
|
|
1098
1294
|
if (newCount > 0)
|
|
1099
1295
|
console.log(` stored ${newCount} new messages`);
|
|
@@ -1110,12 +1306,32 @@ export class ImapManager extends EventEmitter {
|
|
|
1110
1306
|
let deletedCount = 0;
|
|
1111
1307
|
if (!firstSync) {
|
|
1112
1308
|
try {
|
|
1113
|
-
|
|
1309
|
+
// Reuse the server UID list set-diff already fetched.
|
|
1310
|
+
// Without this we made TWO `UID SEARCH` calls per folder
|
|
1311
|
+
// per sync — the first date-bounded (set-diff), the second
|
|
1312
|
+
// `UID SEARCH ALL` (this block). The second call hung
|
|
1313
|
+
// indefinitely on Bob's 134k-message INBOX, blocking the
|
|
1314
|
+
// ops worker and preventing Sent / other folders from
|
|
1315
|
+
// ever getting their turn.
|
|
1316
|
+
const serverUidsArr = serverUidsCached ?? await client.getUids(folder.path);
|
|
1114
1317
|
const serverUids = new Set(serverUidsArr);
|
|
1115
|
-
const
|
|
1318
|
+
const localUidsAll = this.db.getUidsForFolder(accountId, folderId);
|
|
1319
|
+
// When the server-UID list is date-bounded, we can only
|
|
1320
|
+
// reason about deletions for local UIDs in the same window.
|
|
1321
|
+
// UIDs older than the window's lowest server UID may or may
|
|
1322
|
+
// not still be on the server — we never asked. Treat them
|
|
1323
|
+
// as out-of-scope rather than deletion candidates.
|
|
1324
|
+
let localUids = localUidsAll;
|
|
1325
|
+
if (serverUidsAreDateBounded && serverUidsArr.length > 0) {
|
|
1326
|
+
let minServerUid = serverUidsArr[0];
|
|
1327
|
+
for (let i = 1; i < serverUidsArr.length; i++)
|
|
1328
|
+
if (serverUidsArr[i] < minServerUid)
|
|
1329
|
+
minServerUid = serverUidsArr[i];
|
|
1330
|
+
localUids = localUidsAll.filter(u => u >= minServerUid);
|
|
1331
|
+
}
|
|
1116
1332
|
const toDelete = localUids.filter(uid => !serverUids.has(uid));
|
|
1117
|
-
if (serverUidsArr.length === 0 &&
|
|
1118
|
-
console.log(` [sync] ${accountId}/${folder.path}: reconcile skipped — server UID list empty but local has ${
|
|
1333
|
+
if (serverUidsArr.length === 0 && localUidsAll.length > 0) {
|
|
1334
|
+
console.log(` [sync] ${accountId}/${folder.path}: reconcile skipped — server UID list empty but local has ${localUidsAll.length} (treating as transient)`);
|
|
1119
1335
|
}
|
|
1120
1336
|
else if (localUids.length > 0 && toDelete.length / localUids.length > 0.5) {
|
|
1121
1337
|
console.log(` [sync] ${accountId}/${folder.path}: reconcile REFUSED — would delete ${toDelete.length}/${localUids.length} (${Math.round(toDelete.length / localUids.length * 100)}%) — probably a sync bug, skipping`);
|
|
@@ -1311,14 +1527,39 @@ export class ImapManager extends EventEmitter {
|
|
|
1311
1527
|
const highestUid = this.db.getHighestUid(accountId, folder.id);
|
|
1312
1528
|
if (isTrashChild && highestUid === 0)
|
|
1313
1529
|
return;
|
|
1530
|
+
let clientForDiag = null;
|
|
1314
1531
|
try {
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1532
|
+
// Route syncFolder through the slow-lane queue so prefetch
|
|
1533
|
+
// (also slow lane) and sync take strict turns on the slow
|
|
1534
|
+
// client. Previously syncOne grabbed `getOpsClient` and
|
|
1535
|
+
// ran syncFolder directly OUTSIDE the queue; prefetch
|
|
1536
|
+
// chunks via withConnection raced against it on the same
|
|
1537
|
+
// client. Symptom: prefetch sent `SELECT INBOX` then sync
|
|
1538
|
+
// sent `SELECT Sent/Drafts`, then prefetch's `UID FETCH
|
|
1539
|
+
// <inbox-uids>` ran against Drafts → 0 bodies returned →
|
|
1540
|
+
// prefetch logs "0/N — NOT pruning" but bodies never
|
|
1541
|
+
// download. With C123 the fast lane has its own
|
|
1542
|
+
// independent client, so wrapping sync in slow-lane
|
|
1543
|
+
// withConnection doesn't block click-time body fetches.
|
|
1544
|
+
await this.withConnection(accountId, async (client) => {
|
|
1545
|
+
clientForDiag = client;
|
|
1546
|
+
await this.syncFolder(accountId, folder.id, client);
|
|
1547
|
+
}, { slow: true, timeoutMs: PER_FOLDER_TIMEOUT_MS });
|
|
1320
1548
|
}
|
|
1321
1549
|
catch (e) {
|
|
1550
|
+
// C120: per-folder timeout error appends transport
|
|
1551
|
+
// diagnostics so the [sync] log distinguishes "server
|
|
1552
|
+
// stopped responding" (sinceLastRead high) from "we
|
|
1553
|
+
// never finished writing" (writes climbing without
|
|
1554
|
+
// reads). The withConnection timeout already includes
|
|
1555
|
+
// its own message; we annotate further only for the
|
|
1556
|
+
// timeout path.
|
|
1557
|
+
if (/timeout/i.test(e?.message || "")) {
|
|
1558
|
+
const d = clientForDiag?.transport?.diagnostics;
|
|
1559
|
+
if (d) {
|
|
1560
|
+
e.message = `${e.message} [conn#${d.connId} r=${d.bytesRead}B w=${d.bytesWritten}B writes=${d.writeCount} sinceLastRead=${d.lastReadAt ? Date.now() - d.lastReadAt : -1}ms] folder=${folder.path}`;
|
|
1561
|
+
}
|
|
1562
|
+
}
|
|
1322
1563
|
if (e.responseText?.includes("doesn't exist")) {
|
|
1323
1564
|
this.db.deleteFolder(folder.id);
|
|
1324
1565
|
}
|
|
@@ -2224,13 +2465,18 @@ export class ImapManager extends EventEmitter {
|
|
|
2224
2465
|
}
|
|
2225
2466
|
}
|
|
2226
2467
|
await Promise.all(pending);
|
|
2227
|
-
//
|
|
2228
|
-
//
|
|
2229
|
-
//
|
|
2230
|
-
//
|
|
2231
|
-
//
|
|
2232
|
-
//
|
|
2233
|
-
|
|
2468
|
+
// Prune missing UIDs only when the batch actually
|
|
2469
|
+
// completed AND returned at least one body. Earlier
|
|
2470
|
+
// guards (don't-prune-on-thrown-batch) handled 403 /
|
|
2471
|
+
// 429 / network errors but not the "successful but
|
|
2472
|
+
// empty" case — a Gmail batch endpoint that returns
|
|
2473
|
+
// an HTTP 200 with no inner messages, a parser miss,
|
|
2474
|
+
// or a transient that didn't bubble. Same data-loss
|
|
2475
|
+
// pattern: 0 received → all UIDs pruned. The
|
|
2476
|
+
// set-diff reconcile in syncFolder/syncAccountViaApi
|
|
2477
|
+
// owns deletion via a 30-min grace; defer to it.
|
|
2478
|
+
const someReceived = received.size > 0;
|
|
2479
|
+
if (batchSucceeded && someReceived) {
|
|
2234
2480
|
for (const uid of uidsInFolder) {
|
|
2235
2481
|
if (received.has(uid))
|
|
2236
2482
|
continue;
|
|
@@ -2243,6 +2489,9 @@ export class ImapManager extends EventEmitter {
|
|
|
2243
2489
|
catch { /* ignore */ }
|
|
2244
2490
|
}
|
|
2245
2491
|
}
|
|
2492
|
+
else if (batchSucceeded && !someReceived) {
|
|
2493
|
+
console.error(` [prefetch] ${accountId}/${folder.path}: Gmail batch returned 0/${uidsInFolder.length} bodies — NOT pruning (set-diff reconcile owns deletion). UIDs: ${uidsInFolder.slice(0, 5).join(",")}${uidsInFolder.length > 5 ? "..." : ""}`);
|
|
2494
|
+
}
|
|
2246
2495
|
if (counters.errors >= ERROR_BUDGET)
|
|
2247
2496
|
break;
|
|
2248
2497
|
}
|
|
@@ -2283,6 +2532,17 @@ export class ImapManager extends EventEmitter {
|
|
|
2283
2532
|
const bi = bf?.specialUse === "inbox" ? 0 : 1;
|
|
2284
2533
|
return ai - bi;
|
|
2285
2534
|
});
|
|
2535
|
+
// PREFETCH_CHUNK_SIZE: how many UIDs prefetch holds the ops
|
|
2536
|
+
// connection for before yielding. Smaller = interactive
|
|
2537
|
+
// clicks see less wait when prefetch is busy; larger =
|
|
2538
|
+
// fewer round-trips and less per-chunk SELECT overhead.
|
|
2539
|
+
// 25 sits at the knee of that curve for Dovecot — one
|
|
2540
|
+
// FETCH command per chunk (iflow's `fetchChunkSize: 10`
|
|
2541
|
+
// makes that 3 sub-FETCHes), then we release the queue.
|
|
2542
|
+
// Bob 2026-05-08: a 33-UID prefetch held the queue for
|
|
2543
|
+
// 100 minutes during which every click waited; chunking
|
|
2544
|
+
// gives clicks a window roughly every 5-30 seconds.
|
|
2545
|
+
const PREFETCH_CHUNK_SIZE = 25;
|
|
2286
2546
|
for (const [folderId, uids] of orderedFolders) {
|
|
2287
2547
|
const folder = folders.find(f => f.id === folderId);
|
|
2288
2548
|
if (!folder)
|
|
@@ -2291,57 +2551,80 @@ export class ImapManager extends EventEmitter {
|
|
|
2291
2551
|
console.log(` [prefetch] ${accountId}: skipping ${folder.path} (recent timeouts — cooling down)`);
|
|
2292
2552
|
continue;
|
|
2293
2553
|
}
|
|
2294
|
-
|
|
2295
|
-
|
|
2296
|
-
|
|
2297
|
-
|
|
2298
|
-
|
|
2299
|
-
|
|
2300
|
-
|
|
2301
|
-
|
|
2302
|
-
|
|
2303
|
-
|
|
2304
|
-
|
|
2305
|
-
|
|
2306
|
-
|
|
2307
|
-
|
|
2308
|
-
|
|
2309
|
-
|
|
2310
|
-
|
|
2311
|
-
|
|
2312
|
-
|
|
2313
|
-
|
|
2314
|
-
|
|
2315
|
-
|
|
2316
|
-
|
|
2317
|
-
|
|
2318
|
-
|
|
2319
|
-
|
|
2320
|
-
|
|
2321
|
-
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2326
|
-
|
|
2327
|
-
|
|
2328
|
-
|
|
2329
|
-
|
|
2330
|
-
|
|
2331
|
-
|
|
2332
|
-
|
|
2333
|
-
|
|
2334
|
-
|
|
2335
|
-
|
|
2336
|
-
|
|
2337
|
-
|
|
2338
|
-
|
|
2339
|
-
|
|
2340
|
-
|
|
2341
|
-
|
|
2554
|
+
for (let chunkStart = 0; chunkStart < uids.length; chunkStart += PREFETCH_CHUNK_SIZE) {
|
|
2555
|
+
const chunk = uids.slice(chunkStart, chunkStart + PREFETCH_CHUNK_SIZE);
|
|
2556
|
+
const received = new Set();
|
|
2557
|
+
let batchSucceeded = false;
|
|
2558
|
+
try {
|
|
2559
|
+
// Slow lane: prefetch is the textbook "this
|
|
2560
|
+
// might take a while" case — let interactive
|
|
2561
|
+
// ops slip ahead. Each chunk is its own
|
|
2562
|
+
// withConnection so the queue drains the
|
|
2563
|
+
// fast lane between chunks instead of holding
|
|
2564
|
+
// the connection through the whole folder.
|
|
2565
|
+
await this.withConnection(accountId, async (client) => {
|
|
2566
|
+
const pending = [];
|
|
2567
|
+
await client.fetchBodiesBatch(folder.path, chunk, (uid, source) => {
|
|
2568
|
+
received.add(uid);
|
|
2569
|
+
pending.push((async () => {
|
|
2570
|
+
try {
|
|
2571
|
+
const raw = Buffer.from(source, "utf-8");
|
|
2572
|
+
const bodyPath = await this.bodyStore.putMessage(accountId, folderId, uid, raw);
|
|
2573
|
+
this.db.updateBodyPath(accountId, uid, bodyPath);
|
|
2574
|
+
this.emit("bodyCached", accountId, uid);
|
|
2575
|
+
counters.totalFetched++;
|
|
2576
|
+
madeProgress = true;
|
|
2577
|
+
}
|
|
2578
|
+
catch (e) {
|
|
2579
|
+
console.error(` [prefetch] ${accountId}/${uid}: store write failed: ${e.message}`);
|
|
2580
|
+
}
|
|
2581
|
+
})());
|
|
2582
|
+
});
|
|
2583
|
+
await Promise.all(pending);
|
|
2584
|
+
}, { slow: true });
|
|
2585
|
+
batchSucceeded = true;
|
|
2586
|
+
this.clearFolderErrors(accountId, folder.path);
|
|
2587
|
+
}
|
|
2588
|
+
catch (e) {
|
|
2589
|
+
const msg = String(e?.message || "");
|
|
2590
|
+
console.error(` [prefetch] ${accountId} folder ${folder.path} chunk ${chunkStart / PREFETCH_CHUNK_SIZE}: batch fetch failed: ${msg}`);
|
|
2591
|
+
counters.errors++;
|
|
2592
|
+
this.recordFolderError(accountId, folder.path);
|
|
2593
|
+
if (counters.errors >= ERROR_BUDGET)
|
|
2594
|
+
break;
|
|
2595
|
+
}
|
|
2596
|
+
// Prune missing UIDs only when the batch actually
|
|
2597
|
+
// completed AND returned at least one body. A
|
|
2598
|
+
// "successful but empty" batch (FETCH parser
|
|
2599
|
+
// missed every literal, wrong-folder selected,
|
|
2600
|
+
// mid-stream connection hiccup that didn't
|
|
2601
|
+
// throw) is indistinguishable from "all UIDs
|
|
2602
|
+
// were deleted server-side" without that signal
|
|
2603
|
+
// — and erring toward delete cost Bob ~66
|
|
2604
|
+
// messages on 2026-05-08 (`prefetch] bobma: 0
|
|
2605
|
+
// bodies cached, 66 stale rows pruned`). The
|
|
2606
|
+
// set-diff reconcile in syncFolder is the
|
|
2607
|
+
// authoritative deletion path with a 30-min
|
|
2608
|
+
// grace window; prefetch defers to it.
|
|
2609
|
+
const someReceived = received.size > 0;
|
|
2610
|
+
if (batchSucceeded && someReceived)
|
|
2611
|
+
for (const uid of chunk) {
|
|
2612
|
+
if (received.has(uid))
|
|
2613
|
+
continue;
|
|
2614
|
+
try {
|
|
2615
|
+
this.unlinkBodyFile(accountId, uid, folderId).catch(() => { });
|
|
2616
|
+
this.db.deleteMessage(accountId, uid, "prefetch batch: server didn't return body for queued UID — assumed deleted", "mailx-imap prefetchBodies (IMAP batch)");
|
|
2617
|
+
counters.deleted++;
|
|
2618
|
+
madeProgress = true;
|
|
2619
|
+
}
|
|
2620
|
+
catch { /* ignore */ }
|
|
2342
2621
|
}
|
|
2343
|
-
|
|
2622
|
+
else if (batchSucceeded && !someReceived) {
|
|
2623
|
+
console.error(` [prefetch] ${accountId}/${folder.path}: chunk ${chunkStart}-${chunkStart + chunk.length - 1} returned 0/${chunk.length} bodies — NOT pruning (set-diff reconcile owns deletion). UIDs: ${chunk.slice(0, 5).join(",")}${chunk.length > 5 ? "..." : ""}`);
|
|
2344
2624
|
}
|
|
2625
|
+
}
|
|
2626
|
+
if (counters.errors >= ERROR_BUDGET)
|
|
2627
|
+
break;
|
|
2345
2628
|
}
|
|
2346
2629
|
if (counters.errors >= ERROR_BUDGET) {
|
|
2347
2630
|
console.error(` [prefetch] ${accountId}: stopping after ${counters.errors} errors (${counters.totalFetched} cached, ${counters.deleted} pruned)`);
|
|
@@ -3029,6 +3312,20 @@ export class ImapManager extends EventEmitter {
|
|
|
3029
3312
|
// retry. Foreign hosts are left alone — we have no way to know if their
|
|
3030
3313
|
// process is alive. Cross-host stale recovery is the IMAP-folder path's
|
|
3031
3314
|
// job (sweeper looks at server-side claim flags, not local files).
|
|
3315
|
+
// Stale-claim recovery. A claim is "stale" if any of:
|
|
3316
|
+
// (a) the PID is dead — original owner crashed mid-send
|
|
3317
|
+
// (b) the PID is alive BUT it's not us, and the file mtime is
|
|
3318
|
+
// older than STALE_CLAIM_MS — the OS recycled the PID for
|
|
3319
|
+
// some other process. `process.kill(pid, 0)` returning success
|
|
3320
|
+
// only proves *some* process owns that PID, not that it's
|
|
3321
|
+
// our long-dead mailx daemon. Without the age guard, a
|
|
3322
|
+
// claim survives forever as soon as any other Node process
|
|
3323
|
+
// (statusline, msger, npm) gets the recycled PID. Bob saw
|
|
3324
|
+
// this exact case: `.sending-rmf39-63196` sat in the queue
|
|
3325
|
+
// for 7+ hours because PID 63196 was now an unrelated Node.
|
|
3326
|
+
// (c) it's our PID — never sweep our own claim.
|
|
3327
|
+
const STALE_CLAIM_MS = 3600_000;
|
|
3328
|
+
const myPid = process.pid;
|
|
3032
3329
|
for (const dir of [outboxDir, queuedDir]) {
|
|
3033
3330
|
if (!fs.existsSync(dir))
|
|
3034
3331
|
continue;
|
|
@@ -3040,17 +3337,28 @@ export class ImapManager extends EventEmitter {
|
|
|
3040
3337
|
if (host !== this.hostname)
|
|
3041
3338
|
continue;
|
|
3042
3339
|
const pid = parseInt(pidStr);
|
|
3340
|
+
if (pid === myPid)
|
|
3341
|
+
continue; // it's us
|
|
3043
3342
|
let alive = false;
|
|
3044
3343
|
try {
|
|
3045
3344
|
process.kill(pid, 0);
|
|
3046
3345
|
alive = true;
|
|
3047
3346
|
}
|
|
3048
3347
|
catch { /* dead */ }
|
|
3049
|
-
|
|
3050
|
-
|
|
3348
|
+
let ageMs = Infinity;
|
|
3349
|
+
try {
|
|
3350
|
+
ageMs = Date.now() - fs.statSync(path.join(dir, f)).mtimeMs;
|
|
3351
|
+
}
|
|
3352
|
+
catch { /* */ }
|
|
3353
|
+
// Live PID + recent mtime → assume genuine sibling owner.
|
|
3354
|
+
// Live PID + ancient mtime → PID got recycled, sweep it.
|
|
3355
|
+
// Dead PID → sweep regardless of age.
|
|
3356
|
+
if (alive && ageMs < STALE_CLAIM_MS)
|
|
3357
|
+
continue;
|
|
3051
3358
|
try {
|
|
3052
3359
|
fs.renameSync(path.join(dir, f), path.join(dir, original));
|
|
3053
|
-
|
|
3360
|
+
const reason = alive ? `recycled PID, mtime ${Math.round(ageMs / 60_000)}m old` : "dead PID";
|
|
3361
|
+
console.log(` [outbox] Recovered stale claim ${f} → ${original} (${reason})`);
|
|
3054
3362
|
}
|
|
3055
3363
|
catch { /* ignore */ }
|
|
3056
3364
|
}
|
|
@@ -3351,15 +3659,33 @@ export class ImapManager extends EventEmitter {
|
|
|
3351
3659
|
await client.deleteMessageByUid(outboxFolder.path, uid);
|
|
3352
3660
|
}, { slow: true });
|
|
3353
3661
|
if (sentFolder) {
|
|
3662
|
+
let appendedSentUid = null;
|
|
3354
3663
|
try {
|
|
3355
3664
|
await this.withConnection(accountId, async (client) => {
|
|
3356
|
-
await client.appendMessage(sentFolder.path, source, ["\\Seen"]);
|
|
3665
|
+
appendedSentUid = await client.appendMessage(sentFolder.path, source, ["\\Seen"]);
|
|
3357
3666
|
}, { slow: true });
|
|
3358
|
-
this.syncFolder(accountId, sentFolder.id).catch(() => { });
|
|
3359
3667
|
}
|
|
3360
3668
|
catch (sentErr) {
|
|
3361
3669
|
console.error(` [outbox] Failed to copy to Sent: ${sentErr.message} — message was sent successfully`);
|
|
3362
3670
|
}
|
|
3671
|
+
if (appendedSentUid != null) {
|
|
3672
|
+
// The server's APPENDUID response gave us the exact UID
|
|
3673
|
+
// the message landed at in Sent. Insert the local row
|
|
3674
|
+
// directly from the source we already have — no IMAP
|
|
3675
|
+
// round-trip, no SELECT-then-FETCH dance. The Sent
|
|
3676
|
+
// folder view shows the message immediately. The next
|
|
3677
|
+
// periodic sync will see this UID already present and
|
|
3678
|
+
// no-op. Critical: this means a slow/stuck Sent SELECT
|
|
3679
|
+
// never blocks "show me what I just sent" — that was
|
|
3680
|
+
// the user-visible "where's my sent message?" bug.
|
|
3681
|
+
await this.insertLocalRowFromSource(accountId, sentFolder, appendedSentUid, source, ["\\Seen"])
|
|
3682
|
+
.catch((e) => console.error(` [outbox] Local Sent row insert failed: ${e?.message || e} — falling back to broad sync`));
|
|
3683
|
+
}
|
|
3684
|
+
else {
|
|
3685
|
+
// No APPENDUID — server doesn't support UIDPLUS, or
|
|
3686
|
+
// APPEND itself failed. Fall back to a broad sync.
|
|
3687
|
+
this.syncFolder(accountId, sentFolder.id).catch(() => { });
|
|
3688
|
+
}
|
|
3363
3689
|
this.syncFolder(accountId, outboxFolder.id).catch(() => { });
|
|
3364
3690
|
}
|
|
3365
3691
|
}
|
|
@@ -3720,8 +4046,14 @@ export class ImapManager extends EventEmitter {
|
|
|
3720
4046
|
this.stopPeriodicSync();
|
|
3721
4047
|
this.stopOutboxWorker();
|
|
3722
4048
|
await this.stopWatching();
|
|
3723
|
-
// Disconnect
|
|
3724
|
-
|
|
4049
|
+
// Disconnect persistent connections on both lanes. Use a Set
|
|
4050
|
+
// because fastClients can hold an entry an account doesn't have
|
|
4051
|
+
// in opsClients (e.g. body fetches happened but no sync did).
|
|
4052
|
+
const accountIds = new Set([
|
|
4053
|
+
...this.opsClients.keys(),
|
|
4054
|
+
...this.fastClients.keys(),
|
|
4055
|
+
]);
|
|
4056
|
+
for (const accountId of accountIds) {
|
|
3725
4057
|
await this.disconnectOps(accountId);
|
|
3726
4058
|
}
|
|
3727
4059
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bobfrankston/mailx-imap",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.30",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"types": "index.d.ts",
|
|
@@ -11,11 +11,11 @@
|
|
|
11
11
|
"dependencies": {
|
|
12
12
|
"@bobfrankston/mailx-types": "^0.1.10",
|
|
13
13
|
"@bobfrankston/mailx-settings": "^0.1.13",
|
|
14
|
-
"@bobfrankston/mailx-store": "^0.1.
|
|
15
|
-
"@bobfrankston/iflow-direct": "^0.1.
|
|
16
|
-
"@bobfrankston/tcp-transport": "^0.1.
|
|
17
|
-
"@bobfrankston/smtp-direct": "^0.1.
|
|
18
|
-
"@bobfrankston/mailx-sync": "^0.1.
|
|
14
|
+
"@bobfrankston/mailx-store": "^0.1.15",
|
|
15
|
+
"@bobfrankston/iflow-direct": "^0.1.39",
|
|
16
|
+
"@bobfrankston/tcp-transport": "^0.1.6",
|
|
17
|
+
"@bobfrankston/smtp-direct": "^0.1.8",
|
|
18
|
+
"@bobfrankston/mailx-sync": "^0.1.16",
|
|
19
19
|
"@bobfrankston/oauthsupport": "^1.0.26"
|
|
20
20
|
},
|
|
21
21
|
"repository": {
|
|
@@ -39,11 +39,11 @@
|
|
|
39
39
|
"dependencies": {
|
|
40
40
|
"@bobfrankston/mailx-types": "^0.1.10",
|
|
41
41
|
"@bobfrankston/mailx-settings": "^0.1.13",
|
|
42
|
-
"@bobfrankston/mailx-store": "^0.1.
|
|
43
|
-
"@bobfrankston/iflow-direct": "^0.1.
|
|
44
|
-
"@bobfrankston/tcp-transport": "^0.1.
|
|
45
|
-
"@bobfrankston/smtp-direct": "^0.1.
|
|
46
|
-
"@bobfrankston/mailx-sync": "^0.1.
|
|
42
|
+
"@bobfrankston/mailx-store": "^0.1.15",
|
|
43
|
+
"@bobfrankston/iflow-direct": "^0.1.39",
|
|
44
|
+
"@bobfrankston/tcp-transport": "^0.1.6",
|
|
45
|
+
"@bobfrankston/smtp-direct": "^0.1.8",
|
|
46
|
+
"@bobfrankston/mailx-sync": "^0.1.16",
|
|
47
47
|
"@bobfrankston/oauthsupport": "^1.0.26"
|
|
48
48
|
}
|
|
49
49
|
}
|