@bobfrankston/mailx-imap 0.1.28 → 0.1.30

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/index.d.ts +46 -14
  2. package/index.js +522 -190
  3. package/package.json +11 -11
package/index.d.ts CHANGED
@@ -105,11 +105,18 @@ export declare class ImapManager extends EventEmitter {
105
105
  searchAndFetchOnServer(accountId: string, folderId: number, mailboxPath: string, criteria: any): Promise<number[]>;
106
106
  /** Create a fresh IMAP client for an account (public access for API endpoints) */
107
107
  createPublicClient(accountId: string): Promise<any>;
108
- /** Persistent operational connections — one per account, reused for all operations.
109
- * Body fetch, sync, prefetch, outbox-append, flag/move all serialize through
110
- * this single client per account via withConnection(). The priority lane
111
- * in the queue lets interactive clicks jump ahead of background prefetch. */
108
+ /** Persistent slow-lane operational connection per account. Used by sync,
109
+ * prefetch, outbox-append, large backfills, and any other "this might
110
+ * take a while" operation. */
112
111
  private opsClients;
112
+ /** Lazy-allocated fast-lane client per account (C123). Lives alongside
113
+ * `opsClients` so click-time body fetches and flag toggles don't have
114
+ * to wait behind a multi-minute prefetch / backfill on the slow client.
115
+ * Created on the first fast-lane `withConnection` call; reused while
116
+ * alive; recreated on stale-detect or error-discard like opsClients.
117
+ * Costs +1 IMAP socket per active account, well under any reasonable
118
+ * per-user-IP cap (Dovecot's default is 20). */
119
+ private fastClients;
113
120
  /** Two-lane operation queue per account — interactive ops (body fetch on
114
121
  * click, flag toggle) drain before background ops (sync, prefetch). FIFO
115
122
  * within each lane. The single ops connection means there's never a race
@@ -122,8 +129,16 @@ export declare class ImapManager extends EventEmitter {
122
129
  * case (e.g. bobma + bobma2 both on imap.iecc.com). */
123
130
  private hostSemaphores;
124
131
  private static readonly HOST_PERMITS;
125
- /** Get (or create) the persistent operational connection for an account.
126
- * logout() is wrapped as a no-op so legacy callers don't close it. */
132
+ /** Get (or create) a persistent connection for an account on the named
133
+ * lane. Two lanes today: `slow` (the original `opsClients` map sync,
134
+ * prefetch, backfill, outbox) and `fast` (the C123 `fastClients` map —
135
+ * click-time body fetch, flag toggles, anything user-driven). Same
136
+ * stale-detect + reconnect semantics on both. logout() is wrapped as a
137
+ * no-op so legacy callers don't close the persistent client. */
138
+ private getLaneClient;
139
+ /** Backwards-compat shim — callers outside the queue still ask for the
140
+ * ops client by historical name. New code should prefer withConnection
141
+ * with `slow:true` for slow operations. */
127
142
  private getOpsClient;
128
143
  /** Run an operation on the account's connection — queued, sequential, no concurrency */
129
144
  /** Run an operation against the account's single ops connection. Tasks
@@ -143,9 +158,10 @@ export declare class ImapManager extends EventEmitter {
143
158
  slow?: boolean;
144
159
  timeoutMs?: number;
145
160
  }): Promise<T>;
146
- /** Run the next queued task. Fast lane drains before slow.
147
- * Idempotentsafe to call after each task completes; the running
148
- * flag prevents reentrant draining. */
161
+ /** Run the next queued task on each lane. Fast and slow lanes drain
162
+ * CONCURRENTLYeach on its own connection (C123). FIFO within each
163
+ * lane. The running flag per lane prevents reentrant draining of the
164
+ * same lane. */
149
165
  private drainOpsQueue;
150
166
  /** Acquire one slot of the per-host connection semaphore. Returns a release
151
167
  * function — call exactly once when the socket is closed. Used by
@@ -161,12 +177,14 @@ export declare class ImapManager extends EventEmitter {
161
177
  * `purpose` is a short tag printed alongside the `[conn+]` log so we can
162
178
  * tell which code path (ops/idle/etc.) opened each connection. */
163
179
  private newClient;
164
- /** Force-close every IMAP socket for an account — ops + any lingering
165
- * ones in openClients (e.g. an IDLE watcher in flight). Used during
166
- * account removal and disconnectOps so the server's connection slots
167
- * free immediately rather than waiting for socket idle timeouts. */
180
+ /** Force-close every IMAP socket for an account — both lane clients
181
+ * (ops + fast) plus any lingering ones in openClients (e.g. an IDLE
182
+ * watcher in flight). Used during account removal and disconnectOps
183
+ * so the server's connection slots free immediately rather than
184
+ * waiting for socket idle timeouts. */
168
185
  closeAllClients(accountId: string): Promise<void>;
169
- /** Disconnect the persistent operational connection for an account */
186
+ /** Disconnect the persistent operational connections (both lanes) for
187
+ * an account. */
170
188
  disconnectOps(accountId: string): Promise<void>;
171
189
  /** Legacy entry: returns the shared persistent ops client. Most callers
172
190
  * should be using `withConnection()` instead — that gives proper
@@ -191,6 +209,20 @@ export declare class ImapManager extends EventEmitter {
191
209
  syncFolders(accountId: string, client?: any): Promise<Folder[]>;
192
210
  /** Store a batch of messages to DB immediately — used by onChunk for incremental sync */
193
211
  private storeMessages;
212
+ /** Insert a local row for a message we just APPENDed (Sent / Drafts).
213
+ * Uses the source bytes we already have plus the UID returned by the
214
+ * server's APPENDUID response to build the local row directly — no
215
+ * IMAP SELECT/SEARCH/FETCH round-trip required. This is the user-
216
+ * visible fix for "I sent a message but it doesn't show up in mailx
217
+ * Sent until the broad sync finally completes" (which on slow servers
218
+ * can be minutes — and which Thunderbird sidesteps entirely because
219
+ * it inserts its own row at APPEND time too).
220
+ *
221
+ * Fires the same emits as a normal sync so the UI updates. */
222
+ insertLocalRowFromSource(accountId: string, folder: {
223
+ id: number;
224
+ path: string;
225
+ }, uid: number, source: string, flags: string[]): Promise<void>;
194
226
  /** Sync messages for a specific folder */
195
227
  syncFolder(accountId: string, folderId: number, client?: any): Promise<number>;
196
228
  /** Sync all folders for all accounts */
package/index.js CHANGED
@@ -327,11 +327,18 @@ export class ImapManager extends EventEmitter {
327
327
  // All operations on an account are serialized through an operation queue.
328
328
  // No semaphore, no pool, no per-operation connect/disconnect.
329
329
  // IDLE uses a separate connection (see startWatching).
330
- /** Persistent operational connections — one per account, reused for all operations.
331
- * Body fetch, sync, prefetch, outbox-append, flag/move all serialize through
332
- * this single client per account via withConnection(). The priority lane
333
- * in the queue lets interactive clicks jump ahead of background prefetch. */
330
+ /** Persistent slow-lane operational connection per account. Used by sync,
331
+ * prefetch, outbox-append, large backfills, and any other "this might
332
+ * take a while" operation. */
334
333
  opsClients = new Map();
334
+ /** Lazy-allocated fast-lane client per account (C123). Lives alongside
335
+ * `opsClients` so click-time body fetches and flag toggles don't have
336
+ * to wait behind a multi-minute prefetch / backfill on the slow client.
337
+ * Created on the first fast-lane `withConnection` call; reused while
338
+ * alive; recreated on stale-detect or error-discard like opsClients.
339
+ * Costs +1 IMAP socket per active account, well under any reasonable
340
+ * per-user-IP cap (Dovecot's default is 20). */
341
+ fastClients = new Map();
335
342
  /** Two-lane operation queue per account — interactive ops (body fetch on
336
343
  * click, flag toggle) drain before background ops (sync, prefetch). FIFO
337
344
  * within each lane. The single ops connection means there's never a race
@@ -344,10 +351,15 @@ export class ImapManager extends EventEmitter {
344
351
  * case (e.g. bobma + bobma2 both on imap.iecc.com). */
345
352
  hostSemaphores = new Map();
346
353
  static HOST_PERMITS = 4;
347
- /** Get (or create) the persistent operational connection for an account.
348
- * logout() is wrapped as a no-op so legacy callers don't close it. */
349
- async getOpsClient(accountId) {
350
- let client = this.opsClients.get(accountId);
354
+ /** Get (or create) a persistent connection for an account on the named
355
+ * lane. Two lanes today: `slow` (the original `opsClients` map sync,
356
+ * prefetch, backfill, outbox) and `fast` (the C123 `fastClients` map —
357
+ * click-time body fetch, flag toggles, anything user-driven). Same
358
+ * stale-detect + reconnect semantics on both. logout() is wrapped as a
359
+ * no-op so legacy callers don't close the persistent client. */
360
+ async getLaneClient(accountId, lane) {
361
+ const map = lane === "fast" ? this.fastClients : this.opsClients;
362
+ let client = map.get(accountId);
351
363
  if (client) {
352
364
  // C38: health-check the cached client before returning. If the
353
365
  // underlying socket is dead (Dovecot silently dropped IDLE after
@@ -363,19 +375,25 @@ export class ImapManager extends EventEmitter {
363
375
  await (client._realLogout || client.logout)();
364
376
  }
365
377
  catch { /* */ }
366
- this.opsClients.delete(accountId);
367
- console.log(` [conn] ${accountId}: stale ops client detected in getOpsClient — reconnecting`);
378
+ map.delete(accountId);
379
+ console.log(` [conn] ${accountId}: stale ${lane} client detected — reconnecting`);
368
380
  client = undefined;
369
381
  }
370
- client = await this.newClient(accountId, "ops");
382
+ client = await this.newClient(accountId, lane === "fast" ? "fast" : "ops");
371
383
  // Wrap logout as no-op — this is a persistent connection. The
372
384
  // newClient wrapper's close-counter runs on `_realLogout`.
373
385
  const realLogout = client.logout.bind(client);
374
386
  client.logout = async () => { };
375
387
  client._realLogout = realLogout;
376
- this.opsClients.set(accountId, client);
388
+ map.set(accountId, client);
377
389
  return client;
378
390
  }
391
+ /** Backwards-compat shim — callers outside the queue still ask for the
392
+ * ops client by historical name. New code should prefer withConnection
393
+ * with `slow:true` for slow operations. */
394
+ async getOpsClient(accountId) {
395
+ return this.getLaneClient(accountId, "slow");
396
+ }
379
397
  /** Run an operation on the account's connection — queued, sequential, no concurrency */
380
398
  /** Run an operation against the account's single ops connection. Tasks
381
399
  * queue strictly sequentially per account — only one IMAP command in
@@ -393,12 +411,13 @@ export class ImapManager extends EventEmitter {
393
411
  async withConnection(accountId, fn, opts = {}) {
394
412
  let queue = this.opsQueues.get(accountId);
395
413
  if (!queue) {
396
- queue = { fast: [], slow: [], running: false };
414
+ queue = { fast: [], slow: [], runningFast: false, runningSlow: false };
397
415
  this.opsQueues.set(accountId, queue);
398
416
  }
417
+ const lane = opts.slow ? "slow" : "fast";
399
418
  // Per-task wall-clock cap. Without one, a wedged IMAP command (TCP
400
419
  // half-open, server stalled mid-FETCH) keeps the queue's running flag
401
- // set forever and every subsequent fast-lane task — including the
420
+ // set forever and every subsequent same-lane task — including the
402
421
  // retry button the user just hit — waits behind it. Default is
403
422
  // generous; callers driving user-visible reads pass a tighter value.
404
423
  const timeoutMs = opts.timeoutMs ?? 90_000;
@@ -406,11 +425,11 @@ export class ImapManager extends EventEmitter {
406
425
  const task = async () => {
407
426
  let timer;
408
427
  try {
409
- const client = await this.getOpsClient(accountId);
428
+ const client = await this.getLaneClient(accountId, lane);
410
429
  const result = await Promise.race([
411
430
  fn(client),
412
431
  new Promise((_, rej) => {
413
- timer = setTimeout(() => rej(new Error(`ops timeout after ${Math.round(timeoutMs / 1000)}s — discarding client`)), timeoutMs);
432
+ timer = setTimeout(() => rej(new Error(`${lane}-ops timeout after ${Math.round(timeoutMs / 1000)}s — discarding client`)), timeoutMs);
414
433
  }),
415
434
  ]);
416
435
  clearTimeout(timer);
@@ -418,12 +437,15 @@ export class ImapManager extends EventEmitter {
418
437
  }
419
438
  catch (e) {
420
439
  clearTimeout(timer);
421
- // Discard client on any error — keeping a half-broken
422
- // socket poisoned every subsequent request. Destroy
423
- // synchronously kills the in-flight command's socket so
424
- // the underlying promise rejects and stops holding state.
425
- const stale = this.opsClients.get(accountId);
426
- this.opsClients.delete(accountId);
440
+ // Discard the lane's client on any error — a half-broken
441
+ // socket poisons every subsequent request on that lane.
442
+ // Don't touch the OTHER lane's client; the lanes are
443
+ // independent. Destroy synchronously kills the in-flight
444
+ // command's socket so the underlying promise rejects and
445
+ // stops holding state.
446
+ const map = lane === "fast" ? this.fastClients : this.opsClients;
447
+ const stale = map.get(accountId);
448
+ map.delete(accountId);
427
449
  if (stale) {
428
450
  try {
429
451
  await (stale._realLogout || stale.logout)();
@@ -437,25 +459,33 @@ export class ImapManager extends EventEmitter {
437
459
  reject(e);
438
460
  }
439
461
  };
440
- (opts.slow ? queue.slow : queue.fast).push(task);
462
+ queue[lane].push(task);
441
463
  this.drainOpsQueue(accountId);
442
464
  });
443
465
  }
444
- /** Run the next queued task. Fast lane drains before slow.
445
- * Idempotentsafe to call after each task completes; the running
446
- * flag prevents reentrant draining. */
466
+ /** Run the next queued task on each lane. Fast and slow lanes drain
467
+ * CONCURRENTLYeach on its own connection (C123). FIFO within each
468
+ * lane. The running flag per lane prevents reentrant draining of the
469
+ * same lane. */
447
470
  drainOpsQueue(accountId) {
448
471
  const queue = this.opsQueues.get(accountId);
449
- if (!queue || queue.running)
472
+ if (!queue)
450
473
  return;
451
- const next = queue.fast.shift() || queue.slow.shift();
452
- if (!next)
453
- return;
454
- queue.running = true;
455
- next().finally(() => {
456
- queue.running = false;
457
- this.drainOpsQueue(accountId);
458
- });
474
+ const drainLane = (lane) => {
475
+ const runningKey = lane === "fast" ? "runningFast" : "runningSlow";
476
+ if (queue[runningKey])
477
+ return;
478
+ const next = queue[lane].shift();
479
+ if (!next)
480
+ return;
481
+ queue[runningKey] = true;
482
+ next().finally(() => {
483
+ queue[runningKey] = false;
484
+ drainLane(lane);
485
+ });
486
+ };
487
+ drainLane("fast");
488
+ drainLane("slow");
459
489
  }
460
490
  /** Acquire one slot of the per-host connection semaphore. Returns a release
461
491
  * function — call exactly once when the socket is closed. Used by
@@ -506,7 +536,15 @@ export class ImapManager extends EventEmitter {
506
536
  const releaseHostSlot = await this.acquireHostSlot(host);
507
537
  let client;
508
538
  try {
509
- client = new CompatImapClient(config, this.transportFactory);
539
+ // Verbose IMAP wire trace for ops connections only — that's the
540
+ // lane where commands have been hanging silently with no
541
+ // heartbeat / wall-clock fire / reject. Need to see the actual
542
+ // commands sent and bytes received to pinpoint where the
543
+ // pendingCommand is getting lost. Fast lane (C123) shares the
544
+ // verbose treatment so click-time wedges show too. Other lanes
545
+ // (idle, quickCheck) stay quiet so the log doesn't drown.
546
+ const cfgWithVerbose = (purpose === "ops" || purpose === "fast") ? { ...config, verbose: true } : config;
547
+ client = new CompatImapClient(cfgWithVerbose, this.transportFactory);
510
548
  }
511
549
  catch (e) {
512
550
  releaseHostSlot();
@@ -552,22 +590,25 @@ export class ImapManager extends EventEmitter {
552
590
  }
553
591
  return client;
554
592
  }
555
- /** Force-close every IMAP socket for an account — ops + any lingering
556
- * ones in openClients (e.g. an IDLE watcher in flight). Used during
557
- * account removal and disconnectOps so the server's connection slots
558
- * free immediately rather than waiting for socket idle timeouts. */
593
+ /** Force-close every IMAP socket for an account — both lane clients
594
+ * (ops + fast) plus any lingering ones in openClients (e.g. an IDLE
595
+ * watcher in flight). Used during account removal and disconnectOps
596
+ * so the server's connection slots free immediately rather than
597
+ * waiting for socket idle timeouts. */
559
598
  async closeAllClients(accountId) {
560
- const ops = this.opsClients.get(accountId);
561
- this.opsClients.delete(accountId);
562
- if (ops) {
563
- try {
564
- await (ops._realLogout || ops.logout)();
565
- }
566
- catch { /* */ }
567
- try {
568
- ops.destroy?.();
599
+ for (const map of [this.opsClients, this.fastClients]) {
600
+ const c = map.get(accountId);
601
+ map.delete(accountId);
602
+ if (c) {
603
+ try {
604
+ await (c._realLogout || c.logout)();
605
+ }
606
+ catch { /* */ }
607
+ try {
608
+ c.destroy?.();
609
+ }
610
+ catch { /* */ }
569
611
  }
570
- catch { /* */ }
571
612
  }
572
613
  const open = this.openClients.get(accountId);
573
614
  if (open) {
@@ -584,23 +625,26 @@ export class ImapManager extends EventEmitter {
584
625
  open.clear();
585
626
  }
586
627
  }
587
- /** Disconnect the persistent operational connection for an account */
628
+ /** Disconnect the persistent operational connections (both lanes) for
629
+ * an account. */
588
630
  async disconnectOps(accountId) {
589
- const client = this.opsClients.get(accountId);
590
- this.opsClients.delete(accountId);
591
- if (client) {
592
- // Force-close: don't wait for LOGOUT on a possibly dead socket
593
- try {
594
- const timeout = new Promise(r => setTimeout(r, 2000));
595
- await Promise.race([(client._realLogout || client.logout)(), timeout]);
596
- }
597
- catch { /* */ }
598
- // Destroy underlying socket if still open
599
- try {
600
- client.destroy?.();
631
+ for (const [name, map] of [["ops", this.opsClients], ["fast", this.fastClients]]) {
632
+ const client = map.get(accountId);
633
+ map.delete(accountId);
634
+ if (client) {
635
+ // Force-close: don't wait for LOGOUT on a possibly dead socket
636
+ try {
637
+ const timeout = new Promise(r => setTimeout(r, 2000));
638
+ await Promise.race([(client._realLogout || client.logout)(), timeout]);
639
+ }
640
+ catch { /* */ }
641
+ // Destroy underlying socket if still open
642
+ try {
643
+ client.destroy?.();
644
+ }
645
+ catch { /* */ }
646
+ console.log(` [conn] ${accountId} (${name}): disconnected`);
601
647
  }
602
- catch { /* */ }
603
- console.log(` [conn] ${accountId}: disconnected`);
604
648
  }
605
649
  }
606
650
  /** Legacy entry: returns the shared persistent ops client. Most callers
@@ -884,6 +928,49 @@ export class ImapManager extends EventEmitter {
884
928
  }
885
929
  return stored;
886
930
  }
931
+ /** Insert a local row for a message we just APPENDed (Sent / Drafts).
932
+ * Uses the source bytes we already have plus the UID returned by the
933
+ * server's APPENDUID response to build the local row directly — no
934
+ * IMAP SELECT/SEARCH/FETCH round-trip required. This is the user-
935
+ * visible fix for "I sent a message but it doesn't show up in mailx
936
+ * Sent until the broad sync finally completes" (which on slow servers
937
+ * can be minutes — and which Thunderbird sidesteps entirely because
938
+ * it inserts its own row at APPEND time too).
939
+ *
940
+ * Fires the same emits as a normal sync so the UI updates. */
941
+ async insertLocalRowFromSource(accountId, folder, uid, source, flags) {
942
+ const { simpleParser } = await import("mailparser");
943
+ const parsed = await simpleParser(source);
944
+ // Coerce mailparser AddressObject(s) into the flat `{name, address}[]`
945
+ // shape storeMessages's downstream toEmailAddresses expects.
946
+ const flat = (a) => {
947
+ if (!a)
948
+ return [];
949
+ if (Array.isArray(a))
950
+ return a.flatMap(x => x?.value || []);
951
+ return a.value || [];
952
+ };
953
+ const msg = {
954
+ uid,
955
+ messageId: parsed.messageId || "",
956
+ inReplyTo: parsed.inReplyTo || "",
957
+ date: parsed.date || new Date(),
958
+ subject: parsed.subject || "",
959
+ from: flat(parsed.from),
960
+ to: flat(parsed.to),
961
+ cc: flat(parsed.cc),
962
+ seen: flags.includes("\\Seen"),
963
+ flagged: flags.includes("\\Flagged"),
964
+ answered: flags.includes("\\Answered"),
965
+ draft: flags.includes("\\Draft"),
966
+ source,
967
+ size: source.length,
968
+ };
969
+ await this.storeMessages(accountId, folder.id, folder, [msg], 0);
970
+ this.db.recalcFolderCounts(folder.id);
971
+ this.emit("folderCountsChanged", accountId, {});
972
+ console.log(` [local-insert] ${folder.path} UID ${uid}: ${parsed.subject || "(no subject)"} (no IMAP roundtrip)`);
973
+ }
887
974
  /** Sync messages for a specific folder */
888
975
  async syncFolder(accountId, folderId, client) {
889
976
  if (!client)
@@ -894,17 +981,72 @@ export class ImapManager extends EventEmitter {
894
981
  if (!folder)
895
982
  throw new Error(`Folder ${folderId} not found`);
896
983
  this.emit("syncProgress", accountId, `sync:${folder.path}`, 0);
984
+ // Top-of-syncFolder breadcrumb — fires BEFORE any async I/O so a
985
+ // hang in STATUS or SELECT can be located by its absence of a
986
+ // following completion log. Without this, "Sent never showed up
987
+ // in the log" was indistinguishable from "Sent hadn't started"
988
+ // versus "Sent's STATUS call is hung."
989
+ const __sfStart = Date.now();
990
+ console.log(` [sync-enter] ${accountId}/${folder.path} (folderId=${folderId})`);
897
991
  // Get the highest UID we already have for this folder. IMAP UIDs are
898
992
  // monotonically increasing within a UIDVALIDITY (RFC 3501); a
899
993
  // high-water mark is the right anchor for incremental fetch.
900
994
  const highestUid = this.db.getHighestUid(accountId, folderId);
901
- // STATUS-before-SELECT was here. Removed — added a round-trip per
902
- // folder with no measured benefit on Bob's link, and the speculative
903
- // "skip SELECT when nothing changed" optimization didn't actually
904
- // skip anything in practice (server count vs local count nearly
905
- // always differs slightly because of in-flight deletes/moves).
995
+ const localCount = this.db.getMessageCount(accountId, folderId);
996
+ // STATUS-before-SELECT fast O(1) command that returns UIDNEXT and
997
+ // MESSAGES without touching the mailbox SELECT state. RFC 3501 §6.3.10.
998
+ // This is the difference between "INBOX works, Sent doesn't" on this
999
+ // server: INBOX's quickImapCheck uses STATUS first; non-INBOX folders
1000
+ // were going straight to SELECT every cycle, and SELECT on Sent has
1001
+ // been observed wedging this server's ops connection for minutes
1002
+ // (Thunderbird, which uses STATUS-before-SELECT, doesn't see this).
1003
+ // When the server's UIDNEXT-1 ≤ our highestUid AND MESSAGES count
1004
+ // matches our local count, NOTHING changed — skip SELECT entirely.
1005
+ // For first sync (highestUid = 0) skip the STATUS check and let the
1006
+ // existing first-sync path (sequence-FETCH the latest N) run.
1007
+ if (highestUid > 0) {
1008
+ try {
1009
+ console.log(` [sync-status] ${accountId}/${folder.path}: calling STATUS...`);
1010
+ const __statusT0 = Date.now();
1011
+ const status = await client.native?.getStatus?.(folder.path)
1012
+ ?? await client.getStatus?.(folder.path);
1013
+ console.log(` [sync-status] ${accountId}/${folder.path}: STATUS returned in ${Date.now() - __statusT0}ms`);
1014
+ if (status && typeof status.uidNext === "number") {
1015
+ const serverHighest = status.uidNext - 1;
1016
+ const noNewUids = serverHighest <= highestUid;
1017
+ const countsMatch = typeof status.messages === "number" && status.messages === localCount;
1018
+ if (noNewUids && countsMatch) {
1019
+ // Nothing new server-side AND counts match — full
1020
+ // sync would be a no-op. Skip SELECT/SEARCH/FETCH
1021
+ // entirely. This is the path Sent takes most of the
1022
+ // time after the optimistic-APPEND-insert (1.0.573)
1023
+ // already filed the just-sent row locally.
1024
+ console.log(` [sync] ${accountId}/${folder.path}: STATUS clean (uidNext=${status.uidNext} messages=${status.messages}) — skipping SELECT`);
1025
+ this.emit("syncProgress", accountId, `sync:${folder.path}`, 100);
1026
+ return 0;
1027
+ }
1028
+ // Counts differ or server has new UIDs — fall through to
1029
+ // the full sync path below, which knows how to reconcile.
1030
+ console.log(` [sync] ${accountId}/${folder.path}: STATUS uidNext=${status.uidNext} messages=${status.messages} local=${localCount}/${highestUid} — proceeding to SELECT`);
1031
+ }
1032
+ }
1033
+ catch (statusErr) {
1034
+ // STATUS failed — fall through to SELECT (some servers don't
1035
+ // support STATUS on the currently-SELECTed mailbox; per
1036
+ // RFC 3501 §6.3.10 it's actually allowed everywhere but a
1037
+ // misbehaving server might refuse). The fallback is the
1038
+ // pre-1.0.574 behavior, so worst case we lose the speedup.
1039
+ console.log(` [sync] ${accountId}/${folder.path}: STATUS failed (${statusErr?.message || statusErr}) — falling back to SELECT`);
1040
+ }
1041
+ }
906
1042
  console.log(` [sync] ${accountId}/${folder.path}: highestUid=${highestUid}, fetching...`);
907
1043
  let messages;
1044
+ // Cache the server-UID list across set-diff and deletion-recon so we
1045
+ // don't pay for two `UID SEARCH` round-trips (the second one was
1046
+ // `UID SEARCH ALL` which on a 134k-message INBOX hangs for minutes
1047
+ // on this server). Set-diff populates this; deletion-recon reuses.
1048
+ let serverUidsCached = null;
1049
+ let serverUidsAreDateBounded = false;
908
1050
  const firstSync = highestUid === 0;
909
1051
  const historyDays = getHistoryDays(accountId);
910
1052
  // historyDays=0 means "all". On first sync we still cap at 30 days
@@ -922,60 +1064,105 @@ export class ImapManager extends EventEmitter {
922
1064
  const fetched = await client.fetchMessagesSinceUid(folder.path, highestUid, { source: false });
923
1065
  // Filter out the last known message (IMAP * always returns at least one)
924
1066
  messages = fetched.filter((m) => m.uid > highestUid);
925
- // Gap detection: check for missing UIDs within the range we've already synced
926
- // Only reconcile between our lowest and highest UID — don't try to fetch the entire folder history
1067
+ // Reconciliation by SET DIFFERENCE, not high-water-mark.
1068
+ //
1069
+ // The high-water-mark approach (fetch UIDs > highestUid) only
1070
+ // catches NEW arrivals. It cannot see anything below the local
1071
+ // lowest UID — so messages that exist on the server but predate
1072
+ // the user's first sync (or any other gap) are invisible forever.
1073
+ // The previous gap-fill clamped reconciliation to [lowestUid,
1074
+ // highestUid] which made that gap permanent.
1075
+ //
1076
+ // The right model: server UID set is the truth. Anything on
1077
+ // the server that's not local → fetch. Anything local that's
1078
+ // not on the server → reconcile-delete (handled separately by
1079
+ // the deferred-delete path, with grace + 50% safeguards).
1080
+ //
1081
+ // Bounded by a 5000-cap so a misbehaving server response or
1082
+ // brand-new account doesn't try to pull tens of thousands of
1083
+ // historical messages in one cycle. Hit the cap and the next
1084
+ // sync picks up the rest.
927
1085
  const existingUids = this.db.getUidsForFolder(accountId, folderId);
928
- if (existingUids.length > 0) {
929
- try {
930
- const lowestUid = Math.min(...existingUids);
931
- // Fetch UIDs in our known range from IMAP
932
- const rangeUids = await client.getUids(folder.path);
933
- const rangeInScope = rangeUids.filter((uid) => uid >= lowestUid && uid <= highestUid);
934
- const existingSet = new Set(existingUids);
935
- const newSet = new Set(messages.map(m => m.uid));
936
- const missingUids = rangeInScope.filter((uid) => !existingSet.has(uid) && !newSet.has(uid));
937
- if (missingUids.length > 0 && missingUids.length <= 5000) {
938
- console.log(` ${folder.path}: gap detected ${missingUids.length} missing UIDs in range ${lowestUid}..${highestUid}`);
939
- const chunkSize = 500;
940
- let recoveredTotal = 0;
941
- for (let i = 0; i < missingUids.length; i += chunkSize) {
942
- const chunk = missingUids.slice(i, i + chunkSize);
943
- const range = chunk.join(",");
944
- const recovered = await client.fetchMessages(folder.path, range, { source: false });
945
- messages.push(...recovered);
946
- recoveredTotal += recovered.length;
947
- console.log(` ${folder.path}: gap-fill ${recoveredTotal}/${missingUids.length}`);
948
- }
949
- }
950
- else if (missingUids.length > 5000) {
951
- console.log(` ${folder.path}: ${missingUids.length} missing UIDs — too many, skipping reconciliation (delete DB to force full re-sync)`);
1086
+ try {
1087
+ // Date-bound the server UID query when we have a history
1088
+ // window. UID SEARCH ALL on a 134k-message INBOX returns
1089
+ // 134k integers and triggers a heavyweight full-folder
1090
+ // scan on the server; UID SEARCH SINCE <date> returns only
1091
+ // UIDs in the window we actually care about. For
1092
+ // historyDays=0 ("keep everything") we still bound to the
1093
+ // last 5 years on incremental syncs — enough that we never
1094
+ // miss an in-flight message, without enumerating decades of
1095
+ // archive every cycle. First sync gets all UIDs (no anchor
1096
+ // yet so we have to compare against the empty local set
1097
+ // anyway).
1098
+ const SINCE_DAYS = effectiveDays > 0 ? effectiveDays : 1825;
1099
+ const sinceDate = new Date(Date.now() - SINCE_DAYS * 86400000);
1100
+ console.log(` [sync-uids] ${accountId}/${folder.path}: getUidsSince ${sinceDate.toISOString().slice(0, 10)}...`);
1101
+ const __uidsT0 = Date.now();
1102
+ const allServerUids = typeof client.getUidsSince === "function"
1103
+ ? await client.getUidsSince(folder.path, sinceDate)
1104
+ : await client.getUids(folder.path);
1105
+ console.log(` [sync-uids] ${accountId}/${folder.path}: ${allServerUids.length} UIDs in ${Date.now() - __uidsT0}ms`);
1106
+ // Stash for the deletion-reconciliation block below — we
1107
+ // already have the date-bounded server UID list, no point
1108
+ // hitting the server a second time with UID SEARCH ALL.
1109
+ serverUidsCached = allServerUids;
1110
+ serverUidsAreDateBounded = true;
1111
+ const existingSet = new Set(existingUids);
1112
+ const newSet = new Set(messages.map(m => m.uid));
1113
+ const missingUids = allServerUids.filter((uid) => !existingSet.has(uid) && !newSet.has(uid));
1114
+ // Backfill chunk size. Use the passed-in `client` directly
1115
+ // (NOT a nested withConnection) — syncFolder is now wrapped
1116
+ // in withConnection at the call site, so the slow lane is
1117
+ // already locked for our duration. A nested withConnection
1118
+ // would deadlock waiting for the slot we hold.
1119
+ const BACKFILL_CHUNK_SIZE = 100;
1120
+ if (missingUids.length > 0 && missingUids.length <= 5000) {
1121
+ let minU = existingUids[0] ?? 0;
1122
+ for (let i = 1; i < existingUids.length; i++)
1123
+ if (existingUids[i] < minU)
1124
+ minU = existingUids[i];
1125
+ console.log(` ${folder.path}: ${missingUids.length} server-only UIDs (local lowest=${minU}, highest=${highestUid}) — fetching`);
1126
+ let recoveredTotal = 0;
1127
+ for (let i = 0; i < missingUids.length; i += BACKFILL_CHUNK_SIZE) {
1128
+ const chunk = missingUids.slice(i, i + BACKFILL_CHUNK_SIZE);
1129
+ const range = chunk.join(",");
1130
+ const recovered = await client.fetchMessages(folder.path, range, { source: false });
1131
+ messages.push(...recovered);
1132
+ recoveredTotal += recovered.length;
1133
+ console.log(` ${folder.path}: fetch ${recoveredTotal}/${missingUids.length}`);
952
1134
  }
953
1135
  }
954
- catch (e) {
955
- console.error(` ${folder.path}: gap detection failed: ${e.message}`);
956
- }
957
- }
958
- // Backfill: if the history window reaches further back than our
959
- // oldest local message, fetch the gap. Chunk 90 days per sync
960
- // cycle so historyDays=0 catches up incrementally instead of
961
- // asking Dovecot for SEARCH SINCE 1970 in one go.
962
- const oldestDate = this.db.getOldestDate(accountId, folderId);
963
- if (oldestDate > 0 && startDate.getTime() < oldestDate) {
964
- try {
965
- const CHUNK_MS = 90 * 86400000;
966
- const chunkStart = new Date(Math.max(startDate.getTime(), oldestDate - CHUNK_MS));
967
- const existingUids = new Set(this.db.getUidsForFolder(accountId, folderId));
968
- const backfill = await client.fetchMessageByDate(folder.path, chunkStart, new Date(oldestDate), { source: false });
969
- const newBackfill = backfill.filter((m) => !existingUids.has(m.uid));
970
- if (newBackfill.length > 0) {
971
- console.log(` ${folder.path}: backfilling ${newBackfill.length} older messages (${chunkStart.toISOString().slice(0, 10)} → ${new Date(oldestDate).toISOString().slice(0, 10)})`);
972
- messages.push(...newBackfill);
1136
+ else if (missingUids.length > 5000) {
1137
+ console.log(` ${folder.path}: ${missingUids.length} server-only UIDs — capped; will resume next cycle`);
1138
+ // Vanilla IMAP under stable UIDVALIDITY: higher UID =
1139
+ // later assignment ≈ more recent message (Dovecot/
1140
+ // Cyrus). Gmail-API path is separate (no temporal
1141
+ // meaning to its hash-UIDs).
1142
+ const cappedSlice = missingUids.sort((a, b) => b - a).slice(0, 5000);
1143
+ let recoveredTotal = 0;
1144
+ for (let i = 0; i < cappedSlice.length; i += BACKFILL_CHUNK_SIZE) {
1145
+ const chunk = cappedSlice.slice(i, i + BACKFILL_CHUNK_SIZE);
1146
+ const recovered = await client.fetchMessages(folder.path, chunk.join(","), { source: false });
1147
+ messages.push(...recovered);
1148
+ recoveredTotal += recovered.length;
1149
+ console.log(` ${folder.path}: fetch ${recoveredTotal}/5000 (capped)`);
973
1150
  }
974
1151
  }
975
- catch (e) {
976
- console.error(` ${folder.path}: backfill failed: ${e.message}`);
977
- }
978
1152
  }
1153
+ catch (e) {
1154
+ console.error(` ${folder.path}: reconciliation failed: ${e.message}`);
1155
+ }
1156
+ // Date-based backfill via SEARCH SINCE was here. Removed —
1157
+ // set-diff above already covers the same case (any server UID
1158
+ // not in our local set gets fetched, regardless of whether it's
1159
+ // above or below our local highest/lowest). SEARCH SINCE on a
1160
+ // large Dovecot folder walks INTERNALDATE on every message and
1161
+ // can take many minutes, which was wedging the whole syncFolder
1162
+ // call AFTER set-diff had succeeded — preventing syncAccount
1163
+ // from ever moving past INBOX to Sent / other folders. Don't
1164
+ // need both, and date-based-with-high-water-mark is exactly
1165
+ // the brittle model the set-diff replaces.
979
1166
  }
980
1167
  else {
981
1168
  // First sync: fetch in chunks, store each chunk immediately for instant UI
@@ -1094,6 +1281,15 @@ export class ImapManager extends EventEmitter {
1094
1281
  [folderId]: { total: folderInfo?.totalCount || 0, unread: folderInfo?.unreadCount || 0 }
1095
1282
  });
1096
1283
  }
1284
+ // Yield to the event loop between batches so a pending IPC
1285
+ // (e.g. user clicked a message) gets dispatched. Without this
1286
+ // yield, a 5000-message store loop blocked Node's event loop
1287
+ // for ~5s of synchronous SQLite work — clicks fired during
1288
+ // that window felt frozen because the getMessage IPC sat in
1289
+ // stdin until the loop finished. setImmediate runs after I/O
1290
+ // callbacks, which means stdin readable events fire first and
1291
+ // get serviced.
1292
+ await new Promise(r => setImmediate(r));
1097
1293
  }
1098
1294
  if (newCount > 0)
1099
1295
  console.log(` stored ${newCount} new messages`);
@@ -1110,12 +1306,32 @@ export class ImapManager extends EventEmitter {
1110
1306
  let deletedCount = 0;
1111
1307
  if (!firstSync) {
1112
1308
  try {
1113
- const serverUidsArr = await client.getUids(folder.path);
1309
+ // Reuse the server UID list set-diff already fetched.
1310
+ // Without this we made TWO `UID SEARCH` calls per folder
1311
+ // per sync — the first date-bounded (set-diff), the second
1312
+ // `UID SEARCH ALL` (this block). The second call hung
1313
+ // indefinitely on Bob's 134k-message INBOX, blocking the
1314
+ // ops worker and preventing Sent / other folders from
1315
+ // ever getting their turn.
1316
+ const serverUidsArr = serverUidsCached ?? await client.getUids(folder.path);
1114
1317
  const serverUids = new Set(serverUidsArr);
1115
- const localUids = this.db.getUidsForFolder(accountId, folderId);
1318
+ const localUidsAll = this.db.getUidsForFolder(accountId, folderId);
1319
+ // When the server-UID list is date-bounded, we can only
1320
+ // reason about deletions for local UIDs in the same window.
1321
+ // UIDs older than the window's lowest server UID may or may
1322
+ // not still be on the server — we never asked. Treat them
1323
+ // as out-of-scope rather than deletion candidates.
1324
+ let localUids = localUidsAll;
1325
+ if (serverUidsAreDateBounded && serverUidsArr.length > 0) {
1326
+ let minServerUid = serverUidsArr[0];
1327
+ for (let i = 1; i < serverUidsArr.length; i++)
1328
+ if (serverUidsArr[i] < minServerUid)
1329
+ minServerUid = serverUidsArr[i];
1330
+ localUids = localUidsAll.filter(u => u >= minServerUid);
1331
+ }
1116
1332
  const toDelete = localUids.filter(uid => !serverUids.has(uid));
1117
- if (serverUidsArr.length === 0 && localUids.length > 0) {
1118
- console.log(` [sync] ${accountId}/${folder.path}: reconcile skipped — server UID list empty but local has ${localUids.length} (treating as transient)`);
1333
+ if (serverUidsArr.length === 0 && localUidsAll.length > 0) {
1334
+ console.log(` [sync] ${accountId}/${folder.path}: reconcile skipped — server UID list empty but local has ${localUidsAll.length} (treating as transient)`);
1119
1335
  }
1120
1336
  else if (localUids.length > 0 && toDelete.length / localUids.length > 0.5) {
1121
1337
  console.log(` [sync] ${accountId}/${folder.path}: reconcile REFUSED — would delete ${toDelete.length}/${localUids.length} (${Math.round(toDelete.length / localUids.length * 100)}%) — probably a sync bug, skipping`);
@@ -1311,14 +1527,39 @@ export class ImapManager extends EventEmitter {
1311
1527
  const highestUid = this.db.getHighestUid(accountId, folder.id);
1312
1528
  if (isTrashChild && highestUid === 0)
1313
1529
  return;
1530
+ let clientForDiag = null;
1314
1531
  try {
1315
- const fresh = await this.getOpsClient(accountId);
1316
- await Promise.race([
1317
- this.syncFolder(accountId, folder.id, fresh),
1318
- new Promise((_, reject) => setTimeout(() => reject(new Error(`per-folder timeout (${PER_FOLDER_TIMEOUT_MS / 1000}s): ${folder.path}`)), PER_FOLDER_TIMEOUT_MS)),
1319
- ]);
1532
+ // Route syncFolder through the slow-lane queue so prefetch
1533
+ // (also slow lane) and sync take strict turns on the slow
1534
+ // client. Previously syncOne grabbed `getOpsClient` and
1535
+ // ran syncFolder directly OUTSIDE the queue; prefetch
1536
+ // chunks via withConnection raced against it on the same
1537
+ // client. Symptom: prefetch sent `SELECT INBOX` then sync
1538
+ // sent `SELECT Sent/Drafts`, then prefetch's `UID FETCH
1539
+ // <inbox-uids>` ran against Drafts → 0 bodies returned →
1540
+ // prefetch logs "0/N — NOT pruning" but bodies never
1541
+ // download. With C123 the fast lane has its own
1542
+ // independent client, so wrapping sync in slow-lane
1543
+ // withConnection doesn't block click-time body fetches.
1544
+ await this.withConnection(accountId, async (client) => {
1545
+ clientForDiag = client;
1546
+ await this.syncFolder(accountId, folder.id, client);
1547
+ }, { slow: true, timeoutMs: PER_FOLDER_TIMEOUT_MS });
1320
1548
  }
1321
1549
  catch (e) {
1550
+ // C120: per-folder timeout error appends transport
1551
+ // diagnostics so the [sync] log distinguishes "server
1552
+ // stopped responding" (sinceLastRead high) from "we
1553
+ // never finished writing" (writes climbing without
1554
+ // reads). The withConnection timeout already includes
1555
+ // its own message; we annotate further only for the
1556
+ // timeout path.
1557
+ if (/timeout/i.test(e?.message || "")) {
1558
+ const d = clientForDiag?.transport?.diagnostics;
1559
+ if (d) {
1560
+ e.message = `${e.message} [conn#${d.connId} r=${d.bytesRead}B w=${d.bytesWritten}B writes=${d.writeCount} sinceLastRead=${d.lastReadAt ? Date.now() - d.lastReadAt : -1}ms] folder=${folder.path}`;
1561
+ }
1562
+ }
1322
1563
  if (e.responseText?.includes("doesn't exist")) {
1323
1564
  this.db.deleteFolder(folder.id);
1324
1565
  }
@@ -2224,13 +2465,18 @@ export class ImapManager extends EventEmitter {
2224
2465
  }
2225
2466
  }
2226
2467
  await Promise.all(pending);
2227
- // CRITICAL: only prune as "server-deleted" when the batch
2228
- // actually completed. If the batch threw (403, 429, network
2229
- // error, etc.) NOTHING was received, and treating every
2230
- // requested UID as deleted silently wipes 100 messages per
2231
- // batch. That's a data-loss bug. Earlier version did this
2232
- // and pruned 296 messages on a 403 auth error.
2233
- if (batchSucceeded) {
2468
+ // Prune missing UIDs only when the batch actually
2469
+ // completed AND returned at least one body. Earlier
2470
+ // guards (don't-prune-on-thrown-batch) handled 403 /
2471
+ // 429 / network errors but not the "successful but
2472
+ // empty" case a Gmail batch endpoint that returns
2473
+ // an HTTP 200 with no inner messages, a parser miss,
2474
+ // or a transient that didn't bubble. Same data-loss
2475
+ // pattern: 0 received → all UIDs pruned. The
2476
+ // set-diff reconcile in syncFolder/syncAccountViaApi
2477
+ // owns deletion via a 30-min grace; defer to it.
2478
+ const someReceived = received.size > 0;
2479
+ if (batchSucceeded && someReceived) {
2234
2480
  for (const uid of uidsInFolder) {
2235
2481
  if (received.has(uid))
2236
2482
  continue;
@@ -2243,6 +2489,9 @@ export class ImapManager extends EventEmitter {
2243
2489
  catch { /* ignore */ }
2244
2490
  }
2245
2491
  }
2492
+ else if (batchSucceeded && !someReceived) {
2493
+ console.error(` [prefetch] ${accountId}/${folder.path}: Gmail batch returned 0/${uidsInFolder.length} bodies — NOT pruning (set-diff reconcile owns deletion). UIDs: ${uidsInFolder.slice(0, 5).join(",")}${uidsInFolder.length > 5 ? "..." : ""}`);
2494
+ }
2246
2495
  if (counters.errors >= ERROR_BUDGET)
2247
2496
  break;
2248
2497
  }
@@ -2283,6 +2532,17 @@ export class ImapManager extends EventEmitter {
2283
2532
  const bi = bf?.specialUse === "inbox" ? 0 : 1;
2284
2533
  return ai - bi;
2285
2534
  });
2535
+ // PREFETCH_CHUNK_SIZE: how many UIDs prefetch holds the ops
2536
+ // connection for before yielding. Smaller = interactive
2537
+ // clicks see less wait when prefetch is busy; larger =
2538
+ // fewer round-trips and less per-chunk SELECT overhead.
2539
+ // 25 sits at the knee of that curve for Dovecot — one
2540
+ // FETCH command per chunk (iflow's `fetchChunkSize: 10`
2541
+ // makes that 3 sub-FETCHes), then we release the queue.
2542
+ // Bob 2026-05-08: a 33-UID prefetch held the queue for
2543
+ // 100 minutes during which every click waited; chunking
2544
+ // gives clicks a window roughly every 5-30 seconds.
2545
+ const PREFETCH_CHUNK_SIZE = 25;
2286
2546
  for (const [folderId, uids] of orderedFolders) {
2287
2547
  const folder = folders.find(f => f.id === folderId);
2288
2548
  if (!folder)
@@ -2291,57 +2551,80 @@ export class ImapManager extends EventEmitter {
2291
2551
  console.log(` [prefetch] ${accountId}: skipping ${folder.path} (recent timeouts — cooling down)`);
2292
2552
  continue;
2293
2553
  }
2294
- const received = new Set();
2295
- let batchSucceeded = false;
2296
- try {
2297
- // Slow lane: prefetch is the textbook "this might take
2298
- // a while" case — let interactive ops slip ahead.
2299
- await this.withConnection(accountId, async (client) => {
2300
- const pending = [];
2301
- await client.fetchBodiesBatch(folder.path, uids, (uid, source) => {
2302
- received.add(uid);
2303
- pending.push((async () => {
2304
- try {
2305
- const raw = Buffer.from(source, "utf-8");
2306
- const bodyPath = await this.bodyStore.putMessage(accountId, folderId, uid, raw);
2307
- this.db.updateBodyPath(accountId, uid, bodyPath);
2308
- this.emit("bodyCached", accountId, uid);
2309
- counters.totalFetched++;
2310
- madeProgress = true;
2311
- }
2312
- catch (e) {
2313
- console.error(` [prefetch] ${accountId}/${uid}: store write failed: ${e.message}`);
2314
- }
2315
- })());
2316
- });
2317
- await Promise.all(pending);
2318
- }, { slow: true });
2319
- batchSucceeded = true;
2320
- this.clearFolderErrors(accountId, folder.path);
2321
- }
2322
- catch (e) {
2323
- const msg = String(e?.message || "");
2324
- console.error(` [prefetch] ${accountId} folder ${folder.path}: batch fetch failed: ${msg}`);
2325
- counters.errors++;
2326
- this.recordFolderError(accountId, folder.path);
2327
- if (counters.errors >= ERROR_BUDGET)
2328
- break;
2329
- }
2330
- // CRITICAL: only prune when the batch actually completed.
2331
- // A thrown batch means NOTHING was received; treating
2332
- // absence as server-deletion lost 296 messages once.
2333
- if (batchSucceeded)
2334
- for (const uid of uids) {
2335
- if (received.has(uid))
2336
- continue;
2337
- try {
2338
- this.unlinkBodyFile(accountId, uid, folderId).catch(() => { });
2339
- this.db.deleteMessage(accountId, uid, "prefetch batch: server didn't return body for queued UID — assumed deleted", "mailx-imap prefetchBodies (IMAP batch)");
2340
- counters.deleted++;
2341
- madeProgress = true;
2554
+ for (let chunkStart = 0; chunkStart < uids.length; chunkStart += PREFETCH_CHUNK_SIZE) {
2555
+ const chunk = uids.slice(chunkStart, chunkStart + PREFETCH_CHUNK_SIZE);
2556
+ const received = new Set();
2557
+ let batchSucceeded = false;
2558
+ try {
2559
+ // Slow lane: prefetch is the textbook "this
2560
+ // might take a while" case — let interactive
2561
+ // ops slip ahead. Each chunk is its own
2562
+ // withConnection so the queue drains the
2563
+ // fast lane between chunks instead of holding
2564
+ // the connection through the whole folder.
2565
+ await this.withConnection(accountId, async (client) => {
2566
+ const pending = [];
2567
+ await client.fetchBodiesBatch(folder.path, chunk, (uid, source) => {
2568
+ received.add(uid);
2569
+ pending.push((async () => {
2570
+ try {
2571
+ const raw = Buffer.from(source, "utf-8");
2572
+ const bodyPath = await this.bodyStore.putMessage(accountId, folderId, uid, raw);
2573
+ this.db.updateBodyPath(accountId, uid, bodyPath);
2574
+ this.emit("bodyCached", accountId, uid);
2575
+ counters.totalFetched++;
2576
+ madeProgress = true;
2577
+ }
2578
+ catch (e) {
2579
+ console.error(` [prefetch] ${accountId}/${uid}: store write failed: ${e.message}`);
2580
+ }
2581
+ })());
2582
+ });
2583
+ await Promise.all(pending);
2584
+ }, { slow: true });
2585
+ batchSucceeded = true;
2586
+ this.clearFolderErrors(accountId, folder.path);
2587
+ }
2588
+ catch (e) {
2589
+ const msg = String(e?.message || "");
2590
+ console.error(` [prefetch] ${accountId} folder ${folder.path} chunk ${chunkStart / PREFETCH_CHUNK_SIZE}: batch fetch failed: ${msg}`);
2591
+ counters.errors++;
2592
+ this.recordFolderError(accountId, folder.path);
2593
+ if (counters.errors >= ERROR_BUDGET)
2594
+ break;
2595
+ }
2596
+ // Prune missing UIDs only when the batch actually
2597
+ // completed AND returned at least one body. A
2598
+ // "successful but empty" batch (FETCH parser
2599
+ // missed every literal, wrong-folder selected,
2600
+ // mid-stream connection hiccup that didn't
2601
+ // throw) is indistinguishable from "all UIDs
2602
+ // were deleted server-side" without that signal
2603
+ // — and erring toward delete cost Bob ~66
2604
+ // messages on 2026-05-08 (`prefetch] bobma: 0
2605
+ // bodies cached, 66 stale rows pruned`). The
2606
+ // set-diff reconcile in syncFolder is the
2607
+ // authoritative deletion path with a 30-min
2608
+ // grace window; prefetch defers to it.
2609
+ const someReceived = received.size > 0;
2610
+ if (batchSucceeded && someReceived)
2611
+ for (const uid of chunk) {
2612
+ if (received.has(uid))
2613
+ continue;
2614
+ try {
2615
+ this.unlinkBodyFile(accountId, uid, folderId).catch(() => { });
2616
+ this.db.deleteMessage(accountId, uid, "prefetch batch: server didn't return body for queued UID — assumed deleted", "mailx-imap prefetchBodies (IMAP batch)");
2617
+ counters.deleted++;
2618
+ madeProgress = true;
2619
+ }
2620
+ catch { /* ignore */ }
2342
2621
  }
2343
- catch { /* ignore */ }
2622
+ else if (batchSucceeded && !someReceived) {
2623
+ console.error(` [prefetch] ${accountId}/${folder.path}: chunk ${chunkStart}-${chunkStart + chunk.length - 1} returned 0/${chunk.length} bodies — NOT pruning (set-diff reconcile owns deletion). UIDs: ${chunk.slice(0, 5).join(",")}${chunk.length > 5 ? "..." : ""}`);
2344
2624
  }
2625
+ }
2626
+ if (counters.errors >= ERROR_BUDGET)
2627
+ break;
2345
2628
  }
2346
2629
  if (counters.errors >= ERROR_BUDGET) {
2347
2630
  console.error(` [prefetch] ${accountId}: stopping after ${counters.errors} errors (${counters.totalFetched} cached, ${counters.deleted} pruned)`);
@@ -3029,6 +3312,20 @@ export class ImapManager extends EventEmitter {
3029
3312
  // retry. Foreign hosts are left alone — we have no way to know if their
3030
3313
  // process is alive. Cross-host stale recovery is the IMAP-folder path's
3031
3314
  // job (sweeper looks at server-side claim flags, not local files).
3315
+ // Stale-claim recovery. A claim is "stale" if any of:
3316
+ // (a) the PID is dead — original owner crashed mid-send
3317
+ // (b) the PID is alive BUT it's not us, and the file mtime is
3318
+ // older than STALE_CLAIM_MS — the OS recycled the PID for
3319
+ // some other process. `process.kill(pid, 0)` returning success
3320
+ // only proves *some* process owns that PID, not that it's
3321
+ // our long-dead mailx daemon. Without the age guard, a
3322
+ // claim survives forever as soon as any other Node process
3323
+ // (statusline, msger, npm) gets the recycled PID. Bob saw
3324
+ // this exact case: `.sending-rmf39-63196` sat in the queue
3325
+ // for 7+ hours because PID 63196 was now an unrelated Node.
3326
+ // (c) it's our PID — never sweep our own claim.
3327
+ const STALE_CLAIM_MS = 3600_000;
3328
+ const myPid = process.pid;
3032
3329
  for (const dir of [outboxDir, queuedDir]) {
3033
3330
  if (!fs.existsSync(dir))
3034
3331
  continue;
@@ -3040,17 +3337,28 @@ export class ImapManager extends EventEmitter {
3040
3337
  if (host !== this.hostname)
3041
3338
  continue;
3042
3339
  const pid = parseInt(pidStr);
3340
+ if (pid === myPid)
3341
+ continue; // it's us
3043
3342
  let alive = false;
3044
3343
  try {
3045
3344
  process.kill(pid, 0);
3046
3345
  alive = true;
3047
3346
  }
3048
3347
  catch { /* dead */ }
3049
- if (alive)
3050
- continue; // live claim — owner (sibling or self) still has it
3348
+ let ageMs = Infinity;
3349
+ try {
3350
+ ageMs = Date.now() - fs.statSync(path.join(dir, f)).mtimeMs;
3351
+ }
3352
+ catch { /* */ }
3353
+ // Live PID + recent mtime → assume genuine sibling owner.
3354
+ // Live PID + ancient mtime → PID got recycled, sweep it.
3355
+ // Dead PID → sweep regardless of age.
3356
+ if (alive && ageMs < STALE_CLAIM_MS)
3357
+ continue;
3051
3358
  try {
3052
3359
  fs.renameSync(path.join(dir, f), path.join(dir, original));
3053
- console.log(` [outbox] Recovered stale claim ${f} ${original}`);
3360
+ const reason = alive ? `recycled PID, mtime ${Math.round(ageMs / 60_000)}m old` : "dead PID";
3361
+ console.log(` [outbox] Recovered stale claim ${f} → ${original} (${reason})`);
3054
3362
  }
3055
3363
  catch { /* ignore */ }
3056
3364
  }
@@ -3351,15 +3659,33 @@ export class ImapManager extends EventEmitter {
3351
3659
  await client.deleteMessageByUid(outboxFolder.path, uid);
3352
3660
  }, { slow: true });
3353
3661
  if (sentFolder) {
3662
+ let appendedSentUid = null;
3354
3663
  try {
3355
3664
  await this.withConnection(accountId, async (client) => {
3356
- await client.appendMessage(sentFolder.path, source, ["\\Seen"]);
3665
+ appendedSentUid = await client.appendMessage(sentFolder.path, source, ["\\Seen"]);
3357
3666
  }, { slow: true });
3358
- this.syncFolder(accountId, sentFolder.id).catch(() => { });
3359
3667
  }
3360
3668
  catch (sentErr) {
3361
3669
  console.error(` [outbox] Failed to copy to Sent: ${sentErr.message} — message was sent successfully`);
3362
3670
  }
3671
+ if (appendedSentUid != null) {
3672
+ // The server's APPENDUID response gave us the exact UID
3673
+ // the message landed at in Sent. Insert the local row
3674
+ // directly from the source we already have — no IMAP
3675
+ // round-trip, no SELECT-then-FETCH dance. The Sent
3676
+ // folder view shows the message immediately. The next
3677
+ // periodic sync will see this UID already present and
3678
+ // no-op. Critical: this means a slow/stuck Sent SELECT
3679
+ // never blocks "show me what I just sent" — that was
3680
+ // the user-visible "where's my sent message?" bug.
3681
+ await this.insertLocalRowFromSource(accountId, sentFolder, appendedSentUid, source, ["\\Seen"])
3682
+ .catch((e) => console.error(` [outbox] Local Sent row insert failed: ${e?.message || e} — falling back to broad sync`));
3683
+ }
3684
+ else {
3685
+ // No APPENDUID — server doesn't support UIDPLUS, or
3686
+ // APPEND itself failed. Fall back to a broad sync.
3687
+ this.syncFolder(accountId, sentFolder.id).catch(() => { });
3688
+ }
3363
3689
  this.syncFolder(accountId, outboxFolder.id).catch(() => { });
3364
3690
  }
3365
3691
  }
@@ -3720,8 +4046,14 @@ export class ImapManager extends EventEmitter {
3720
4046
  this.stopPeriodicSync();
3721
4047
  this.stopOutboxWorker();
3722
4048
  await this.stopWatching();
3723
- // Disconnect all persistent operational connections
3724
- for (const [accountId] of this.opsClients) {
4049
+ // Disconnect persistent connections on both lanes. Use a Set
4050
+ // because fastClients can hold an entry an account doesn't have
4051
+ // in opsClients (e.g. body fetches happened but no sync did).
4052
+ const accountIds = new Set([
4053
+ ...this.opsClients.keys(),
4054
+ ...this.fastClients.keys(),
4055
+ ]);
4056
+ for (const accountId of accountIds) {
3725
4057
  await this.disconnectOps(accountId);
3726
4058
  }
3727
4059
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@bobfrankston/mailx-imap",
3
- "version": "0.1.28",
3
+ "version": "0.1.30",
4
4
  "type": "module",
5
5
  "main": "index.js",
6
6
  "types": "index.d.ts",
@@ -11,11 +11,11 @@
11
11
  "dependencies": {
12
12
  "@bobfrankston/mailx-types": "^0.1.10",
13
13
  "@bobfrankston/mailx-settings": "^0.1.13",
14
- "@bobfrankston/mailx-store": "^0.1.13",
15
- "@bobfrankston/iflow-direct": "^0.1.35",
16
- "@bobfrankston/tcp-transport": "^0.1.5",
17
- "@bobfrankston/smtp-direct": "^0.1.5",
18
- "@bobfrankston/mailx-sync": "^0.1.15",
14
+ "@bobfrankston/mailx-store": "^0.1.15",
15
+ "@bobfrankston/iflow-direct": "^0.1.39",
16
+ "@bobfrankston/tcp-transport": "^0.1.6",
17
+ "@bobfrankston/smtp-direct": "^0.1.8",
18
+ "@bobfrankston/mailx-sync": "^0.1.16",
19
19
  "@bobfrankston/oauthsupport": "^1.0.26"
20
20
  },
21
21
  "repository": {
@@ -39,11 +39,11 @@
39
39
  "dependencies": {
40
40
  "@bobfrankston/mailx-types": "^0.1.10",
41
41
  "@bobfrankston/mailx-settings": "^0.1.13",
42
- "@bobfrankston/mailx-store": "^0.1.13",
43
- "@bobfrankston/iflow-direct": "^0.1.35",
44
- "@bobfrankston/tcp-transport": "^0.1.5",
45
- "@bobfrankston/smtp-direct": "^0.1.5",
46
- "@bobfrankston/mailx-sync": "^0.1.15",
42
+ "@bobfrankston/mailx-store": "^0.1.15",
43
+ "@bobfrankston/iflow-direct": "^0.1.39",
44
+ "@bobfrankston/tcp-transport": "^0.1.6",
45
+ "@bobfrankston/smtp-direct": "^0.1.8",
46
+ "@bobfrankston/mailx-sync": "^0.1.16",
47
47
  "@bobfrankston/oauthsupport": "^1.0.26"
48
48
  }
49
49
  }