@bobfrankston/mailx-imap 0.1.28 → 0.1.29

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/index.d.ts +46 -14
  2. package/index.js +527 -187
  3. package/package.json +11 -11
package/index.d.ts CHANGED
@@ -105,11 +105,18 @@ export declare class ImapManager extends EventEmitter {
105
105
  searchAndFetchOnServer(accountId: string, folderId: number, mailboxPath: string, criteria: any): Promise<number[]>;
106
106
  /** Create a fresh IMAP client for an account (public access for API endpoints) */
107
107
  createPublicClient(accountId: string): Promise<any>;
108
- /** Persistent operational connections — one per account, reused for all operations.
109
- * Body fetch, sync, prefetch, outbox-append, flag/move all serialize through
110
- * this single client per account via withConnection(). The priority lane
111
- * in the queue lets interactive clicks jump ahead of background prefetch. */
108
+ /** Persistent slow-lane operational connection per account. Used by sync,
109
+ * prefetch, outbox-append, large backfills, and any other "this might
110
+ * take a while" operation. */
112
111
  private opsClients;
112
+ /** Lazy-allocated fast-lane client per account (C123). Lives alongside
113
+ * `opsClients` so click-time body fetches and flag toggles don't have
114
+ * to wait behind a multi-minute prefetch / backfill on the slow client.
115
+ * Created on the first fast-lane `withConnection` call; reused while
116
+ * alive; recreated on stale-detect or error-discard like opsClients.
117
+ * Costs +1 IMAP socket per active account, well under any reasonable
118
+ * per-user-IP cap (Dovecot's default is 20). */
119
+ private fastClients;
113
120
  /** Two-lane operation queue per account — interactive ops (body fetch on
114
121
  * click, flag toggle) drain before background ops (sync, prefetch). FIFO
115
122
  * within each lane. The single ops connection means there's never a race
@@ -122,8 +129,16 @@ export declare class ImapManager extends EventEmitter {
122
129
  * case (e.g. bobma + bobma2 both on imap.iecc.com). */
123
130
  private hostSemaphores;
124
131
  private static readonly HOST_PERMITS;
125
- /** Get (or create) the persistent operational connection for an account.
126
- * logout() is wrapped as a no-op so legacy callers don't close it. */
132
+ /** Get (or create) a persistent connection for an account on the named
133
+ * lane. Two lanes today: `slow` (the original `opsClients` map sync,
134
+ * prefetch, backfill, outbox) and `fast` (the C123 `fastClients` map —
135
+ * click-time body fetch, flag toggles, anything user-driven). Same
136
+ * stale-detect + reconnect semantics on both. logout() is wrapped as a
137
+ * no-op so legacy callers don't close the persistent client. */
138
+ private getLaneClient;
139
+ /** Backwards-compat shim — callers outside the queue still ask for the
140
+ * ops client by historical name. New code should prefer withConnection
141
+ * with `slow:true` for slow operations. */
127
142
  private getOpsClient;
128
143
  /** Run an operation on the account's connection — queued, sequential, no concurrency */
129
144
  /** Run an operation against the account's single ops connection. Tasks
@@ -143,9 +158,10 @@ export declare class ImapManager extends EventEmitter {
143
158
  slow?: boolean;
144
159
  timeoutMs?: number;
145
160
  }): Promise<T>;
146
- /** Run the next queued task. Fast lane drains before slow.
147
- * Idempotentsafe to call after each task completes; the running
148
- * flag prevents reentrant draining. */
161
+ /** Run the next queued task on each lane. Fast and slow lanes drain
162
+ * CONCURRENTLYeach on its own connection (C123). FIFO within each
163
+ * lane. The running flag per lane prevents reentrant draining of the
164
+ * same lane. */
149
165
  private drainOpsQueue;
150
166
  /** Acquire one slot of the per-host connection semaphore. Returns a release
151
167
  * function — call exactly once when the socket is closed. Used by
@@ -161,12 +177,14 @@ export declare class ImapManager extends EventEmitter {
161
177
  * `purpose` is a short tag printed alongside the `[conn+]` log so we can
162
178
  * tell which code path (ops/idle/etc.) opened each connection. */
163
179
  private newClient;
164
- /** Force-close every IMAP socket for an account — ops + any lingering
165
- * ones in openClients (e.g. an IDLE watcher in flight). Used during
166
- * account removal and disconnectOps so the server's connection slots
167
- * free immediately rather than waiting for socket idle timeouts. */
180
+ /** Force-close every IMAP socket for an account — both lane clients
181
+ * (ops + fast) plus any lingering ones in openClients (e.g. an IDLE
182
+ * watcher in flight). Used during account removal and disconnectOps
183
+ * so the server's connection slots free immediately rather than
184
+ * waiting for socket idle timeouts. */
168
185
  closeAllClients(accountId: string): Promise<void>;
169
- /** Disconnect the persistent operational connection for an account */
186
+ /** Disconnect the persistent operational connections (both lanes) for
187
+ * an account. */
170
188
  disconnectOps(accountId: string): Promise<void>;
171
189
  /** Legacy entry: returns the shared persistent ops client. Most callers
172
190
  * should be using `withConnection()` instead — that gives proper
@@ -191,6 +209,20 @@ export declare class ImapManager extends EventEmitter {
191
209
  syncFolders(accountId: string, client?: any): Promise<Folder[]>;
192
210
  /** Store a batch of messages to DB immediately — used by onChunk for incremental sync */
193
211
  private storeMessages;
212
+ /** Insert a local row for a message we just APPENDed (Sent / Drafts).
213
+ * Uses the source bytes we already have plus the UID returned by the
214
+ * server's APPENDUID response to build the local row directly — no
215
+ * IMAP SELECT/SEARCH/FETCH round-trip required. This is the user-
216
+ * visible fix for "I sent a message but it doesn't show up in mailx
217
+ * Sent until the broad sync finally completes" (which on slow servers
218
+ * can be minutes — and which Thunderbird sidesteps entirely because
219
+ * it inserts its own row at APPEND time too).
220
+ *
221
+ * Fires the same emits as a normal sync so the UI updates. */
222
+ insertLocalRowFromSource(accountId: string, folder: {
223
+ id: number;
224
+ path: string;
225
+ }, uid: number, source: string, flags: string[]): Promise<void>;
194
226
  /** Sync messages for a specific folder */
195
227
  syncFolder(accountId: string, folderId: number, client?: any): Promise<number>;
196
228
  /** Sync all folders for all accounts */
package/index.js CHANGED
@@ -327,11 +327,18 @@ export class ImapManager extends EventEmitter {
327
327
  // All operations on an account are serialized through an operation queue.
328
328
  // No semaphore, no pool, no per-operation connect/disconnect.
329
329
  // IDLE uses a separate connection (see startWatching).
330
- /** Persistent operational connections — one per account, reused for all operations.
331
- * Body fetch, sync, prefetch, outbox-append, flag/move all serialize through
332
- * this single client per account via withConnection(). The priority lane
333
- * in the queue lets interactive clicks jump ahead of background prefetch. */
330
+ /** Persistent slow-lane operational connection per account. Used by sync,
331
+ * prefetch, outbox-append, large backfills, and any other "this might
332
+ * take a while" operation. */
334
333
  opsClients = new Map();
334
+ /** Lazy-allocated fast-lane client per account (C123). Lives alongside
335
+ * `opsClients` so click-time body fetches and flag toggles don't have
336
+ * to wait behind a multi-minute prefetch / backfill on the slow client.
337
+ * Created on the first fast-lane `withConnection` call; reused while
338
+ * alive; recreated on stale-detect or error-discard like opsClients.
339
+ * Costs +1 IMAP socket per active account, well under any reasonable
340
+ * per-user-IP cap (Dovecot's default is 20). */
341
+ fastClients = new Map();
335
342
  /** Two-lane operation queue per account — interactive ops (body fetch on
336
343
  * click, flag toggle) drain before background ops (sync, prefetch). FIFO
337
344
  * within each lane. The single ops connection means there's never a race
@@ -344,10 +351,15 @@ export class ImapManager extends EventEmitter {
344
351
  * case (e.g. bobma + bobma2 both on imap.iecc.com). */
345
352
  hostSemaphores = new Map();
346
353
  static HOST_PERMITS = 4;
347
- /** Get (or create) the persistent operational connection for an account.
348
- * logout() is wrapped as a no-op so legacy callers don't close it. */
349
- async getOpsClient(accountId) {
350
- let client = this.opsClients.get(accountId);
354
+ /** Get (or create) a persistent connection for an account on the named
355
+ * lane. Two lanes today: `slow` (the original `opsClients` map sync,
356
+ * prefetch, backfill, outbox) and `fast` (the C123 `fastClients` map —
357
+ * click-time body fetch, flag toggles, anything user-driven). Same
358
+ * stale-detect + reconnect semantics on both. logout() is wrapped as a
359
+ * no-op so legacy callers don't close the persistent client. */
360
+ async getLaneClient(accountId, lane) {
361
+ const map = lane === "fast" ? this.fastClients : this.opsClients;
362
+ let client = map.get(accountId);
351
363
  if (client) {
352
364
  // C38: health-check the cached client before returning. If the
353
365
  // underlying socket is dead (Dovecot silently dropped IDLE after
@@ -363,19 +375,25 @@ export class ImapManager extends EventEmitter {
363
375
  await (client._realLogout || client.logout)();
364
376
  }
365
377
  catch { /* */ }
366
- this.opsClients.delete(accountId);
367
- console.log(` [conn] ${accountId}: stale ops client detected in getOpsClient — reconnecting`);
378
+ map.delete(accountId);
379
+ console.log(` [conn] ${accountId}: stale ${lane} client detected — reconnecting`);
368
380
  client = undefined;
369
381
  }
370
- client = await this.newClient(accountId, "ops");
382
+ client = await this.newClient(accountId, lane === "fast" ? "fast" : "ops");
371
383
  // Wrap logout as no-op — this is a persistent connection. The
372
384
  // newClient wrapper's close-counter runs on `_realLogout`.
373
385
  const realLogout = client.logout.bind(client);
374
386
  client.logout = async () => { };
375
387
  client._realLogout = realLogout;
376
- this.opsClients.set(accountId, client);
388
+ map.set(accountId, client);
377
389
  return client;
378
390
  }
391
+ /** Backwards-compat shim — callers outside the queue still ask for the
392
+ * ops client by historical name. New code should prefer withConnection
393
+ * with `slow:true` for slow operations. */
394
+ async getOpsClient(accountId) {
395
+ return this.getLaneClient(accountId, "slow");
396
+ }
379
397
  /** Run an operation on the account's connection — queued, sequential, no concurrency */
380
398
  /** Run an operation against the account's single ops connection. Tasks
381
399
  * queue strictly sequentially per account — only one IMAP command in
@@ -393,12 +411,13 @@ export class ImapManager extends EventEmitter {
393
411
  async withConnection(accountId, fn, opts = {}) {
394
412
  let queue = this.opsQueues.get(accountId);
395
413
  if (!queue) {
396
- queue = { fast: [], slow: [], running: false };
414
+ queue = { fast: [], slow: [], runningFast: false, runningSlow: false };
397
415
  this.opsQueues.set(accountId, queue);
398
416
  }
417
+ const lane = opts.slow ? "slow" : "fast";
399
418
  // Per-task wall-clock cap. Without one, a wedged IMAP command (TCP
400
419
  // half-open, server stalled mid-FETCH) keeps the queue's running flag
401
- // set forever and every subsequent fast-lane task — including the
420
+ // set forever and every subsequent same-lane task — including the
402
421
  // retry button the user just hit — waits behind it. Default is
403
422
  // generous; callers driving user-visible reads pass a tighter value.
404
423
  const timeoutMs = opts.timeoutMs ?? 90_000;
@@ -406,11 +425,11 @@ export class ImapManager extends EventEmitter {
406
425
  const task = async () => {
407
426
  let timer;
408
427
  try {
409
- const client = await this.getOpsClient(accountId);
428
+ const client = await this.getLaneClient(accountId, lane);
410
429
  const result = await Promise.race([
411
430
  fn(client),
412
431
  new Promise((_, rej) => {
413
- timer = setTimeout(() => rej(new Error(`ops timeout after ${Math.round(timeoutMs / 1000)}s — discarding client`)), timeoutMs);
432
+ timer = setTimeout(() => rej(new Error(`${lane}-ops timeout after ${Math.round(timeoutMs / 1000)}s — discarding client`)), timeoutMs);
414
433
  }),
415
434
  ]);
416
435
  clearTimeout(timer);
@@ -418,12 +437,15 @@ export class ImapManager extends EventEmitter {
418
437
  }
419
438
  catch (e) {
420
439
  clearTimeout(timer);
421
- // Discard client on any error — keeping a half-broken
422
- // socket poisoned every subsequent request. Destroy
423
- // synchronously kills the in-flight command's socket so
424
- // the underlying promise rejects and stops holding state.
425
- const stale = this.opsClients.get(accountId);
426
- this.opsClients.delete(accountId);
440
+ // Discard the lane's client on any error — a half-broken
441
+ // socket poisons every subsequent request on that lane.
442
+ // Don't touch the OTHER lane's client; the lanes are
443
+ // independent. Destroy synchronously kills the in-flight
444
+ // command's socket so the underlying promise rejects and
445
+ // stops holding state.
446
+ const map = lane === "fast" ? this.fastClients : this.opsClients;
447
+ const stale = map.get(accountId);
448
+ map.delete(accountId);
427
449
  if (stale) {
428
450
  try {
429
451
  await (stale._realLogout || stale.logout)();
@@ -437,25 +459,33 @@ export class ImapManager extends EventEmitter {
437
459
  reject(e);
438
460
  }
439
461
  };
440
- (opts.slow ? queue.slow : queue.fast).push(task);
462
+ queue[lane].push(task);
441
463
  this.drainOpsQueue(accountId);
442
464
  });
443
465
  }
444
- /** Run the next queued task. Fast lane drains before slow.
445
- * Idempotentsafe to call after each task completes; the running
446
- * flag prevents reentrant draining. */
466
+ /** Run the next queued task on each lane. Fast and slow lanes drain
467
+ * CONCURRENTLYeach on its own connection (C123). FIFO within each
468
+ * lane. The running flag per lane prevents reentrant draining of the
469
+ * same lane. */
447
470
  drainOpsQueue(accountId) {
448
471
  const queue = this.opsQueues.get(accountId);
449
- if (!queue || queue.running)
450
- return;
451
- const next = queue.fast.shift() || queue.slow.shift();
452
- if (!next)
472
+ if (!queue)
453
473
  return;
454
- queue.running = true;
455
- next().finally(() => {
456
- queue.running = false;
457
- this.drainOpsQueue(accountId);
458
- });
474
+ const drainLane = (lane) => {
475
+ const runningKey = lane === "fast" ? "runningFast" : "runningSlow";
476
+ if (queue[runningKey])
477
+ return;
478
+ const next = queue[lane].shift();
479
+ if (!next)
480
+ return;
481
+ queue[runningKey] = true;
482
+ next().finally(() => {
483
+ queue[runningKey] = false;
484
+ drainLane(lane);
485
+ });
486
+ };
487
+ drainLane("fast");
488
+ drainLane("slow");
459
489
  }
460
490
  /** Acquire one slot of the per-host connection semaphore. Returns a release
461
491
  * function — call exactly once when the socket is closed. Used by
@@ -506,7 +536,15 @@ export class ImapManager extends EventEmitter {
506
536
  const releaseHostSlot = await this.acquireHostSlot(host);
507
537
  let client;
508
538
  try {
509
- client = new CompatImapClient(config, this.transportFactory);
539
+ // Verbose IMAP wire trace for ops connections only — that's the
540
+ // lane where commands have been hanging silently with no
541
+ // heartbeat / wall-clock fire / reject. Need to see the actual
542
+ // commands sent and bytes received to pinpoint where the
543
+ // pendingCommand is getting lost. Fast lane (C123) shares the
544
+ // verbose treatment so click-time wedges show too. Other lanes
545
+ // (idle, quickCheck) stay quiet so the log doesn't drown.
546
+ const cfgWithVerbose = (purpose === "ops" || purpose === "fast") ? { ...config, verbose: true } : config;
547
+ client = new CompatImapClient(cfgWithVerbose, this.transportFactory);
510
548
  }
511
549
  catch (e) {
512
550
  releaseHostSlot();
@@ -552,22 +590,25 @@ export class ImapManager extends EventEmitter {
552
590
  }
553
591
  return client;
554
592
  }
555
- /** Force-close every IMAP socket for an account — ops + any lingering
556
- * ones in openClients (e.g. an IDLE watcher in flight). Used during
557
- * account removal and disconnectOps so the server's connection slots
558
- * free immediately rather than waiting for socket idle timeouts. */
593
+ /** Force-close every IMAP socket for an account — both lane clients
594
+ * (ops + fast) plus any lingering ones in openClients (e.g. an IDLE
595
+ * watcher in flight). Used during account removal and disconnectOps
596
+ * so the server's connection slots free immediately rather than
597
+ * waiting for socket idle timeouts. */
559
598
  async closeAllClients(accountId) {
560
- const ops = this.opsClients.get(accountId);
561
- this.opsClients.delete(accountId);
562
- if (ops) {
563
- try {
564
- await (ops._realLogout || ops.logout)();
565
- }
566
- catch { /* */ }
567
- try {
568
- ops.destroy?.();
599
+ for (const map of [this.opsClients, this.fastClients]) {
600
+ const c = map.get(accountId);
601
+ map.delete(accountId);
602
+ if (c) {
603
+ try {
604
+ await (c._realLogout || c.logout)();
605
+ }
606
+ catch { /* */ }
607
+ try {
608
+ c.destroy?.();
609
+ }
610
+ catch { /* */ }
569
611
  }
570
- catch { /* */ }
571
612
  }
572
613
  const open = this.openClients.get(accountId);
573
614
  if (open) {
@@ -584,23 +625,26 @@ export class ImapManager extends EventEmitter {
584
625
  open.clear();
585
626
  }
586
627
  }
587
- /** Disconnect the persistent operational connection for an account */
628
+ /** Disconnect the persistent operational connections (both lanes) for
629
+ * an account. */
588
630
  async disconnectOps(accountId) {
589
- const client = this.opsClients.get(accountId);
590
- this.opsClients.delete(accountId);
591
- if (client) {
592
- // Force-close: don't wait for LOGOUT on a possibly dead socket
593
- try {
594
- const timeout = new Promise(r => setTimeout(r, 2000));
595
- await Promise.race([(client._realLogout || client.logout)(), timeout]);
596
- }
597
- catch { /* */ }
598
- // Destroy underlying socket if still open
599
- try {
600
- client.destroy?.();
631
+ for (const [name, map] of [["ops", this.opsClients], ["fast", this.fastClients]]) {
632
+ const client = map.get(accountId);
633
+ map.delete(accountId);
634
+ if (client) {
635
+ // Force-close: don't wait for LOGOUT on a possibly dead socket
636
+ try {
637
+ const timeout = new Promise(r => setTimeout(r, 2000));
638
+ await Promise.race([(client._realLogout || client.logout)(), timeout]);
639
+ }
640
+ catch { /* */ }
641
+ // Destroy underlying socket if still open
642
+ try {
643
+ client.destroy?.();
644
+ }
645
+ catch { /* */ }
646
+ console.log(` [conn] ${accountId} (${name}): disconnected`);
601
647
  }
602
- catch { /* */ }
603
- console.log(` [conn] ${accountId}: disconnected`);
604
648
  }
605
649
  }
606
650
  /** Legacy entry: returns the shared persistent ops client. Most callers
@@ -884,6 +928,49 @@ export class ImapManager extends EventEmitter {
884
928
  }
885
929
  return stored;
886
930
  }
931
+ /** Insert a local row for a message we just APPENDed (Sent / Drafts).
932
+ * Uses the source bytes we already have plus the UID returned by the
933
+ * server's APPENDUID response to build the local row directly — no
934
+ * IMAP SELECT/SEARCH/FETCH round-trip required. This is the user-
935
+ * visible fix for "I sent a message but it doesn't show up in mailx
936
+ * Sent until the broad sync finally completes" (which on slow servers
937
+ * can be minutes — and which Thunderbird sidesteps entirely because
938
+ * it inserts its own row at APPEND time too).
939
+ *
940
+ * Fires the same emits as a normal sync so the UI updates. */
941
+ async insertLocalRowFromSource(accountId, folder, uid, source, flags) {
942
+ const { simpleParser } = await import("mailparser");
943
+ const parsed = await simpleParser(source);
944
+ // Coerce mailparser AddressObject(s) into the flat `{name, address}[]`
945
+ // shape storeMessages's downstream toEmailAddresses expects.
946
+ const flat = (a) => {
947
+ if (!a)
948
+ return [];
949
+ if (Array.isArray(a))
950
+ return a.flatMap(x => x?.value || []);
951
+ return a.value || [];
952
+ };
953
+ const msg = {
954
+ uid,
955
+ messageId: parsed.messageId || "",
956
+ inReplyTo: parsed.inReplyTo || "",
957
+ date: parsed.date || new Date(),
958
+ subject: parsed.subject || "",
959
+ from: flat(parsed.from),
960
+ to: flat(parsed.to),
961
+ cc: flat(parsed.cc),
962
+ seen: flags.includes("\\Seen"),
963
+ flagged: flags.includes("\\Flagged"),
964
+ answered: flags.includes("\\Answered"),
965
+ draft: flags.includes("\\Draft"),
966
+ source,
967
+ size: source.length,
968
+ };
969
+ await this.storeMessages(accountId, folder.id, folder, [msg], 0);
970
+ this.db.recalcFolderCounts(folder.id);
971
+ this.emit("folderCountsChanged", accountId, {});
972
+ console.log(` [local-insert] ${folder.path} UID ${uid}: ${parsed.subject || "(no subject)"} (no IMAP roundtrip)`);
973
+ }
887
974
  /** Sync messages for a specific folder */
888
975
  async syncFolder(accountId, folderId, client) {
889
976
  if (!client)
@@ -894,17 +981,72 @@ export class ImapManager extends EventEmitter {
894
981
  if (!folder)
895
982
  throw new Error(`Folder ${folderId} not found`);
896
983
  this.emit("syncProgress", accountId, `sync:${folder.path}`, 0);
984
+ // Top-of-syncFolder breadcrumb — fires BEFORE any async I/O so a
985
+ // hang in STATUS or SELECT can be located by its absence of a
986
+ // following completion log. Without this, "Sent never showed up
987
+ // in the log" was indistinguishable from "Sent hadn't started"
988
+ // versus "Sent's STATUS call is hung."
989
+ const __sfStart = Date.now();
990
+ console.log(` [sync-enter] ${accountId}/${folder.path} (folderId=${folderId})`);
897
991
  // Get the highest UID we already have for this folder. IMAP UIDs are
898
992
  // monotonically increasing within a UIDVALIDITY (RFC 3501); a
899
993
  // high-water mark is the right anchor for incremental fetch.
900
994
  const highestUid = this.db.getHighestUid(accountId, folderId);
901
- // STATUS-before-SELECT was here. Removed — added a round-trip per
902
- // folder with no measured benefit on Bob's link, and the speculative
903
- // "skip SELECT when nothing changed" optimization didn't actually
904
- // skip anything in practice (server count vs local count nearly
905
- // always differs slightly because of in-flight deletes/moves).
995
+ const localCount = this.db.getMessageCount(accountId, folderId);
996
+ // STATUS-before-SELECT fast O(1) command that returns UIDNEXT and
997
+ // MESSAGES without touching the mailbox SELECT state. RFC 3501 §6.3.10.
998
+ // This is the difference between "INBOX works, Sent doesn't" on this
999
+ // server: INBOX's quickImapCheck uses STATUS first; non-INBOX folders
1000
+ // were going straight to SELECT every cycle, and SELECT on Sent has
1001
+ // been observed wedging this server's ops connection for minutes
1002
+ // (Thunderbird, which uses STATUS-before-SELECT, doesn't see this).
1003
+ // When the server's UIDNEXT-1 ≤ our highestUid AND MESSAGES count
1004
+ // matches our local count, NOTHING changed — skip SELECT entirely.
1005
+ // For first sync (highestUid = 0) skip the STATUS check and let the
1006
+ // existing first-sync path (sequence-FETCH the latest N) run.
1007
+ if (highestUid > 0) {
1008
+ try {
1009
+ console.log(` [sync-status] ${accountId}/${folder.path}: calling STATUS...`);
1010
+ const __statusT0 = Date.now();
1011
+ const status = await client.native?.getStatus?.(folder.path)
1012
+ ?? await client.getStatus?.(folder.path);
1013
+ console.log(` [sync-status] ${accountId}/${folder.path}: STATUS returned in ${Date.now() - __statusT0}ms`);
1014
+ if (status && typeof status.uidNext === "number") {
1015
+ const serverHighest = status.uidNext - 1;
1016
+ const noNewUids = serverHighest <= highestUid;
1017
+ const countsMatch = typeof status.messages === "number" && status.messages === localCount;
1018
+ if (noNewUids && countsMatch) {
1019
+ // Nothing new server-side AND counts match — full
1020
+ // sync would be a no-op. Skip SELECT/SEARCH/FETCH
1021
+ // entirely. This is the path Sent takes most of the
1022
+ // time after the optimistic-APPEND-insert (1.0.573)
1023
+ // already filed the just-sent row locally.
1024
+ console.log(` [sync] ${accountId}/${folder.path}: STATUS clean (uidNext=${status.uidNext} messages=${status.messages}) — skipping SELECT`);
1025
+ this.emit("syncProgress", accountId, `sync:${folder.path}`, 100);
1026
+ return 0;
1027
+ }
1028
+ // Counts differ or server has new UIDs — fall through to
1029
+ // the full sync path below, which knows how to reconcile.
1030
+ console.log(` [sync] ${accountId}/${folder.path}: STATUS uidNext=${status.uidNext} messages=${status.messages} local=${localCount}/${highestUid} — proceeding to SELECT`);
1031
+ }
1032
+ }
1033
+ catch (statusErr) {
1034
+ // STATUS failed — fall through to SELECT (some servers don't
1035
+ // support STATUS on the currently-SELECTed mailbox; per
1036
+ // RFC 3501 §6.3.10 it's actually allowed everywhere but a
1037
+ // misbehaving server might refuse). The fallback is the
1038
+ // pre-1.0.574 behavior, so worst case we lose the speedup.
1039
+ console.log(` [sync] ${accountId}/${folder.path}: STATUS failed (${statusErr?.message || statusErr}) — falling back to SELECT`);
1040
+ }
1041
+ }
906
1042
  console.log(` [sync] ${accountId}/${folder.path}: highestUid=${highestUid}, fetching...`);
907
1043
  let messages;
1044
+ // Cache the server-UID list across set-diff and deletion-recon so we
1045
+ // don't pay for two `UID SEARCH` round-trips (the second one was
1046
+ // `UID SEARCH ALL` which on a 134k-message INBOX hangs for minutes
1047
+ // on this server). Set-diff populates this; deletion-recon reuses.
1048
+ let serverUidsCached = null;
1049
+ let serverUidsAreDateBounded = false;
908
1050
  const firstSync = highestUid === 0;
909
1051
  const historyDays = getHistoryDays(accountId);
910
1052
  // historyDays=0 means "all". On first sync we still cap at 30 days
@@ -922,60 +1064,125 @@ export class ImapManager extends EventEmitter {
922
1064
  const fetched = await client.fetchMessagesSinceUid(folder.path, highestUid, { source: false });
923
1065
  // Filter out the last known message (IMAP * always returns at least one)
924
1066
  messages = fetched.filter((m) => m.uid > highestUid);
925
- // Gap detection: check for missing UIDs within the range we've already synced
926
- // Only reconcile between our lowest and highest UID — don't try to fetch the entire folder history
1067
+ // Reconciliation by SET DIFFERENCE, not high-water-mark.
1068
+ //
1069
+ // The high-water-mark approach (fetch UIDs > highestUid) only
1070
+ // catches NEW arrivals. It cannot see anything below the local
1071
+ // lowest UID — so messages that exist on the server but predate
1072
+ // the user's first sync (or any other gap) are invisible forever.
1073
+ // The previous gap-fill clamped reconciliation to [lowestUid,
1074
+ // highestUid] which made that gap permanent.
1075
+ //
1076
+ // The right model: server UID set is the truth. Anything on
1077
+ // the server that's not local → fetch. Anything local that's
1078
+ // not on the server → reconcile-delete (handled separately by
1079
+ // the deferred-delete path, with grace + 50% safeguards).
1080
+ //
1081
+ // Bounded by a 5000-cap so a misbehaving server response or
1082
+ // brand-new account doesn't try to pull tens of thousands of
1083
+ // historical messages in one cycle. Hit the cap and the next
1084
+ // sync picks up the rest.
927
1085
  const existingUids = this.db.getUidsForFolder(accountId, folderId);
928
- if (existingUids.length > 0) {
929
- try {
930
- const lowestUid = Math.min(...existingUids);
931
- // Fetch UIDs in our known range from IMAP
932
- const rangeUids = await client.getUids(folder.path);
933
- const rangeInScope = rangeUids.filter((uid) => uid >= lowestUid && uid <= highestUid);
934
- const existingSet = new Set(existingUids);
935
- const newSet = new Set(messages.map(m => m.uid));
936
- const missingUids = rangeInScope.filter((uid) => !existingSet.has(uid) && !newSet.has(uid));
937
- if (missingUids.length > 0 && missingUids.length <= 5000) {
938
- console.log(` ${folder.path}: gap detected ${missingUids.length} missing UIDs in range ${lowestUid}..${highestUid}`);
939
- const chunkSize = 500;
940
- let recoveredTotal = 0;
941
- for (let i = 0; i < missingUids.length; i += chunkSize) {
942
- const chunk = missingUids.slice(i, i + chunkSize);
943
- const range = chunk.join(",");
944
- const recovered = await client.fetchMessages(folder.path, range, { source: false });
945
- messages.push(...recovered);
946
- recoveredTotal += recovered.length;
947
- console.log(` ${folder.path}: gap-fill ${recoveredTotal}/${missingUids.length}`);
948
- }
949
- }
950
- else if (missingUids.length > 5000) {
951
- console.log(` ${folder.path}: ${missingUids.length} missing UIDs — too many, skipping reconciliation (delete DB to force full re-sync)`);
1086
+ try {
1087
+ // Date-bound the server UID query when we have a history
1088
+ // window. UID SEARCH ALL on a 134k-message INBOX returns
1089
+ // 134k integers and triggers a heavyweight full-folder
1090
+ // scan on the server; UID SEARCH SINCE <date> returns only
1091
+ // UIDs in the window we actually care about. For
1092
+ // historyDays=0 ("keep everything") we still bound to the
1093
+ // last 5 years on incremental syncs — enough that we never
1094
+ // miss an in-flight message, without enumerating decades of
1095
+ // archive every cycle. First sync gets all UIDs (no anchor
1096
+ // yet so we have to compare against the empty local set
1097
+ // anyway).
1098
+ const SINCE_DAYS = effectiveDays > 0 ? effectiveDays : 1825;
1099
+ const sinceDate = new Date(Date.now() - SINCE_DAYS * 86400000);
1100
+ console.log(` [sync-uids] ${accountId}/${folder.path}: getUidsSince ${sinceDate.toISOString().slice(0, 10)}...`);
1101
+ const __uidsT0 = Date.now();
1102
+ const allServerUids = typeof client.getUidsSince === "function"
1103
+ ? await client.getUidsSince(folder.path, sinceDate)
1104
+ : await client.getUids(folder.path);
1105
+ console.log(` [sync-uids] ${accountId}/${folder.path}: ${allServerUids.length} UIDs in ${Date.now() - __uidsT0}ms`);
1106
+ // Stash for the deletion-reconciliation block below — we
1107
+ // already have the date-bounded server UID list, no point
1108
+ // hitting the server a second time with UID SEARCH ALL.
1109
+ serverUidsCached = allServerUids;
1110
+ serverUidsAreDateBounded = true;
1111
+ const existingSet = new Set(existingUids);
1112
+ const newSet = new Set(messages.map(m => m.uid));
1113
+ const missingUids = allServerUids.filter((uid) => !existingSet.has(uid) && !newSet.has(uid));
1114
+ // Backfill chunk size: ops queue yields between chunks so
1115
+ // a click-time body fetch can interleave. 100 UIDs per
1116
+ // chunk balances throughput (one FETCH command per chunk)
1117
+ // against latency (a chunk takes 1-3s on Dovecot, which
1118
+ // is the worst-case wait an interactive click endures).
1119
+ // The 500-chunk version held the ops queue for the
1120
+ // entire backfill — Bob 2026-05-08 saw a click-to-render
1121
+ // wait of 100+ minutes on a busy backfill of the IP
1122
+ // folder.
1123
+ const BACKFILL_CHUNK_SIZE = 100;
1124
+ if (missingUids.length > 0 && missingUids.length <= 5000) {
1125
+ // For the log line we report a count; computing
1126
+ // min/max via a spread (`Math.min(...arr)`) blows V8's
1127
+ // argument limit on folders with tens of thousands of
1128
+ // UIDs. Use a manual reduce.
1129
+ let minU = existingUids[0] ?? 0;
1130
+ for (let i = 1; i < existingUids.length; i++)
1131
+ if (existingUids[i] < minU)
1132
+ minU = existingUids[i];
1133
+ console.log(` ${folder.path}: ${missingUids.length} server-only UIDs (local lowest=${minU}, highest=${highestUid}) — fetching`);
1134
+ let recoveredTotal = 0;
1135
+ for (let i = 0; i < missingUids.length; i += BACKFILL_CHUNK_SIZE) {
1136
+ const chunk = missingUids.slice(i, i + BACKFILL_CHUNK_SIZE);
1137
+ const range = chunk.join(",");
1138
+ // Each chunk gets its own withConnection slow-lane
1139
+ // turn so any fast-lane click queued in the
1140
+ // meantime gets serviced between chunks. The
1141
+ // outer `client` param is bypassed here; the
1142
+ // queue-managed client is the same persistent
1143
+ // ops client (getOpsClient).
1144
+ const recovered = await this.withConnection(accountId, async (c) => await c.fetchMessages(folder.path, range, { source: false }), { slow: true });
1145
+ messages.push(...recovered);
1146
+ recoveredTotal += recovered.length;
1147
+ console.log(` ${folder.path}: fetch ${recoveredTotal}/${missingUids.length}`);
952
1148
  }
953
1149
  }
954
- catch (e) {
955
- console.error(` ${folder.path}: gap detection failed: ${e.message}`);
956
- }
957
- }
958
- // Backfill: if the history window reaches further back than our
959
- // oldest local message, fetch the gap. Chunk 90 days per sync
960
- // cycle so historyDays=0 catches up incrementally instead of
961
- // asking Dovecot for SEARCH SINCE 1970 in one go.
962
- const oldestDate = this.db.getOldestDate(accountId, folderId);
963
- if (oldestDate > 0 && startDate.getTime() < oldestDate) {
964
- try {
965
- const CHUNK_MS = 90 * 86400000;
966
- const chunkStart = new Date(Math.max(startDate.getTime(), oldestDate - CHUNK_MS));
967
- const existingUids = new Set(this.db.getUidsForFolder(accountId, folderId));
968
- const backfill = await client.fetchMessageByDate(folder.path, chunkStart, new Date(oldestDate), { source: false });
969
- const newBackfill = backfill.filter((m) => !existingUids.has(m.uid));
970
- if (newBackfill.length > 0) {
971
- console.log(` ${folder.path}: backfilling ${newBackfill.length} older messages (${chunkStart.toISOString().slice(0, 10)} → ${new Date(oldestDate).toISOString().slice(0, 10)})`);
972
- messages.push(...newBackfill);
1150
+ else if (missingUids.length > 5000) {
1151
+ console.log(` ${folder.path}: ${missingUids.length} server-only UIDs — capped; will resume next cycle`);
1152
+ // ASSUMPTION: vanilla IMAP under stable UIDVALIDITY,
1153
+ // higher UID = later assignment ≈ more recent message.
1154
+ // True for Dovecot / Cyrus / standard IMAP, which is the
1155
+ // only place this code path runs (Gmail-API mode goes
1156
+ // through syncAccountViaApi and doesn't reach here
1157
+ // its synthesized hash-UIDs have no temporal meaning).
1158
+ // If we ever wire this for a non-monotonic UID source,
1159
+ // sort by date instead but that means an extra
1160
+ // fetch-of-INTERNALDATE round-trip, which we avoid for
1161
+ // free here under the IMAP guarantee.
1162
+ const cappedSlice = missingUids.sort((a, b) => b - a).slice(0, 5000);
1163
+ let recoveredTotal = 0;
1164
+ for (let i = 0; i < cappedSlice.length; i += BACKFILL_CHUNK_SIZE) {
1165
+ const chunk = cappedSlice.slice(i, i + BACKFILL_CHUNK_SIZE);
1166
+ const recovered = await this.withConnection(accountId, async (c) => await c.fetchMessages(folder.path, chunk.join(","), { source: false }), { slow: true });
1167
+ messages.push(...recovered);
1168
+ recoveredTotal += recovered.length;
1169
+ console.log(` ${folder.path}: fetch ${recoveredTotal}/5000 (capped)`);
973
1170
  }
974
1171
  }
975
- catch (e) {
976
- console.error(` ${folder.path}: backfill failed: ${e.message}`);
977
- }
978
1172
  }
1173
+ catch (e) {
1174
+ console.error(` ${folder.path}: reconciliation failed: ${e.message}`);
1175
+ }
1176
+ // Date-based backfill via SEARCH SINCE was here. Removed —
1177
+ // set-diff above already covers the same case (any server UID
1178
+ // not in our local set gets fetched, regardless of whether it's
1179
+ // above or below our local highest/lowest). SEARCH SINCE on a
1180
+ // large Dovecot folder walks INTERNALDATE on every message and
1181
+ // can take many minutes, which was wedging the whole syncFolder
1182
+ // call AFTER set-diff had succeeded — preventing syncAccount
1183
+ // from ever moving past INBOX to Sent / other folders. Don't
1184
+ // need both, and date-based-with-high-water-mark is exactly
1185
+ // the brittle model the set-diff replaces.
979
1186
  }
980
1187
  else {
981
1188
  // First sync: fetch in chunks, store each chunk immediately for instant UI
@@ -1094,6 +1301,15 @@ export class ImapManager extends EventEmitter {
1094
1301
  [folderId]: { total: folderInfo?.totalCount || 0, unread: folderInfo?.unreadCount || 0 }
1095
1302
  });
1096
1303
  }
1304
+ // Yield to the event loop between batches so a pending IPC
1305
+ // (e.g. user clicked a message) gets dispatched. Without this
1306
+ // yield, a 5000-message store loop blocked Node's event loop
1307
+ // for ~5s of synchronous SQLite work — clicks fired during
1308
+ // that window felt frozen because the getMessage IPC sat in
1309
+ // stdin until the loop finished. setImmediate runs after I/O
1310
+ // callbacks, which means stdin readable events fire first and
1311
+ // get serviced.
1312
+ await new Promise(r => setImmediate(r));
1097
1313
  }
1098
1314
  if (newCount > 0)
1099
1315
  console.log(` stored ${newCount} new messages`);
@@ -1110,12 +1326,32 @@ export class ImapManager extends EventEmitter {
1110
1326
  let deletedCount = 0;
1111
1327
  if (!firstSync) {
1112
1328
  try {
1113
- const serverUidsArr = await client.getUids(folder.path);
1329
+ // Reuse the server UID list set-diff already fetched.
1330
+ // Without this we made TWO `UID SEARCH` calls per folder
1331
+ // per sync — the first date-bounded (set-diff), the second
1332
+ // `UID SEARCH ALL` (this block). The second call hung
1333
+ // indefinitely on Bob's 134k-message INBOX, blocking the
1334
+ // ops worker and preventing Sent / other folders from
1335
+ // ever getting their turn.
1336
+ const serverUidsArr = serverUidsCached ?? await client.getUids(folder.path);
1114
1337
  const serverUids = new Set(serverUidsArr);
1115
- const localUids = this.db.getUidsForFolder(accountId, folderId);
1338
+ const localUidsAll = this.db.getUidsForFolder(accountId, folderId);
1339
+ // When the server-UID list is date-bounded, we can only
1340
+ // reason about deletions for local UIDs in the same window.
1341
+ // UIDs older than the window's lowest server UID may or may
1342
+ // not still be on the server — we never asked. Treat them
1343
+ // as out-of-scope rather than deletion candidates.
1344
+ let localUids = localUidsAll;
1345
+ if (serverUidsAreDateBounded && serverUidsArr.length > 0) {
1346
+ let minServerUid = serverUidsArr[0];
1347
+ for (let i = 1; i < serverUidsArr.length; i++)
1348
+ if (serverUidsArr[i] < minServerUid)
1349
+ minServerUid = serverUidsArr[i];
1350
+ localUids = localUidsAll.filter(u => u >= minServerUid);
1351
+ }
1116
1352
  const toDelete = localUids.filter(uid => !serverUids.has(uid));
1117
- if (serverUidsArr.length === 0 && localUids.length > 0) {
1118
- console.log(` [sync] ${accountId}/${folder.path}: reconcile skipped — server UID list empty but local has ${localUids.length} (treating as transient)`);
1353
+ if (serverUidsArr.length === 0 && localUidsAll.length > 0) {
1354
+ console.log(` [sync] ${accountId}/${folder.path}: reconcile skipped — server UID list empty but local has ${localUidsAll.length} (treating as transient)`);
1119
1355
  }
1120
1356
  else if (localUids.length > 0 && toDelete.length / localUids.length > 0.5) {
1121
1357
  console.log(` [sync] ${accountId}/${folder.path}: reconcile REFUSED — would delete ${toDelete.length}/${localUids.length} (${Math.round(toDelete.length / localUids.length * 100)}%) — probably a sync bug, skipping`);
@@ -1311,11 +1547,24 @@ export class ImapManager extends EventEmitter {
1311
1547
  const highestUid = this.db.getHighestUid(accountId, folder.id);
1312
1548
  if (isTrashChild && highestUid === 0)
1313
1549
  return;
1550
+ let fresh = null;
1314
1551
  try {
1315
- const fresh = await this.getOpsClient(accountId);
1552
+ fresh = await this.getOpsClient(accountId);
1316
1553
  await Promise.race([
1317
1554
  this.syncFolder(accountId, folder.id, fresh),
1318
- new Promise((_, reject) => setTimeout(() => reject(new Error(`per-folder timeout (${PER_FOLDER_TIMEOUT_MS / 1000}s): ${folder.path}`)), PER_FOLDER_TIMEOUT_MS)),
1555
+ new Promise((_, reject) => setTimeout(() => {
1556
+ // C120: pull TCP transport diagnostics into the
1557
+ // per-folder timeout error so the [sync] log
1558
+ // distinguishes "server stopped responding"
1559
+ // (sinceLastRead high) from "we never finished
1560
+ // writing" (writes climbing without reads). Same
1561
+ // shape iflow-direct emits on its own timeouts.
1562
+ const d = fresh?.transport?.diagnostics;
1563
+ const diag = d
1564
+ ? ` [conn#${d.connId} r=${d.bytesRead}B w=${d.bytesWritten}B writes=${d.writeCount} sinceLastRead=${d.lastReadAt ? Date.now() - d.lastReadAt : -1}ms]`
1565
+ : "";
1566
+ reject(new Error(`per-folder timeout (${PER_FOLDER_TIMEOUT_MS / 1000}s): ${folder.path}${diag}`));
1567
+ }, PER_FOLDER_TIMEOUT_MS)),
1319
1568
  ]);
1320
1569
  }
1321
1570
  catch (e) {
@@ -2224,13 +2473,18 @@ export class ImapManager extends EventEmitter {
2224
2473
  }
2225
2474
  }
2226
2475
  await Promise.all(pending);
2227
- // CRITICAL: only prune as "server-deleted" when the batch
2228
- // actually completed. If the batch threw (403, 429, network
2229
- // error, etc.) NOTHING was received, and treating every
2230
- // requested UID as deleted silently wipes 100 messages per
2231
- // batch. That's a data-loss bug. Earlier version did this
2232
- // and pruned 296 messages on a 403 auth error.
2233
- if (batchSucceeded) {
2476
+ // Prune missing UIDs only when the batch actually
2477
+ // completed AND returned at least one body. Earlier
2478
+ // guards (don't-prune-on-thrown-batch) handled 403 /
2479
+ // 429 / network errors but not the "successful but
2480
+ // empty" case a Gmail batch endpoint that returns
2481
+ // an HTTP 200 with no inner messages, a parser miss,
2482
+ // or a transient that didn't bubble. Same data-loss
2483
+ // pattern: 0 received → all UIDs pruned. The
2484
+ // set-diff reconcile in syncFolder/syncAccountViaApi
2485
+ // owns deletion via a 30-min grace; defer to it.
2486
+ const someReceived = received.size > 0;
2487
+ if (batchSucceeded && someReceived) {
2234
2488
  for (const uid of uidsInFolder) {
2235
2489
  if (received.has(uid))
2236
2490
  continue;
@@ -2243,6 +2497,9 @@ export class ImapManager extends EventEmitter {
2243
2497
  catch { /* ignore */ }
2244
2498
  }
2245
2499
  }
2500
+ else if (batchSucceeded && !someReceived) {
2501
+ console.error(` [prefetch] ${accountId}/${folder.path}: Gmail batch returned 0/${uidsInFolder.length} bodies — NOT pruning (set-diff reconcile owns deletion). UIDs: ${uidsInFolder.slice(0, 5).join(",")}${uidsInFolder.length > 5 ? "..." : ""}`);
2502
+ }
2246
2503
  if (counters.errors >= ERROR_BUDGET)
2247
2504
  break;
2248
2505
  }
@@ -2283,6 +2540,17 @@ export class ImapManager extends EventEmitter {
2283
2540
  const bi = bf?.specialUse === "inbox" ? 0 : 1;
2284
2541
  return ai - bi;
2285
2542
  });
2543
+ // PREFETCH_CHUNK_SIZE: how many UIDs prefetch holds the ops
2544
+ // connection for before yielding. Smaller = interactive
2545
+ // clicks see less wait when prefetch is busy; larger =
2546
+ // fewer round-trips and less per-chunk SELECT overhead.
2547
+ // 25 sits at the knee of that curve for Dovecot — one
2548
+ // FETCH command per chunk (iflow's `fetchChunkSize: 10`
2549
+ // makes that 3 sub-FETCHes), then we release the queue.
2550
+ // Bob 2026-05-08: a 33-UID prefetch held the queue for
2551
+ // 100 minutes during which every click waited; chunking
2552
+ // gives clicks a window roughly every 5-30 seconds.
2553
+ const PREFETCH_CHUNK_SIZE = 25;
2286
2554
  for (const [folderId, uids] of orderedFolders) {
2287
2555
  const folder = folders.find(f => f.id === folderId);
2288
2556
  if (!folder)
@@ -2291,57 +2559,80 @@ export class ImapManager extends EventEmitter {
2291
2559
  console.log(` [prefetch] ${accountId}: skipping ${folder.path} (recent timeouts — cooling down)`);
2292
2560
  continue;
2293
2561
  }
2294
- const received = new Set();
2295
- let batchSucceeded = false;
2296
- try {
2297
- // Slow lane: prefetch is the textbook "this might take
2298
- // a while" case — let interactive ops slip ahead.
2299
- await this.withConnection(accountId, async (client) => {
2300
- const pending = [];
2301
- await client.fetchBodiesBatch(folder.path, uids, (uid, source) => {
2302
- received.add(uid);
2303
- pending.push((async () => {
2304
- try {
2305
- const raw = Buffer.from(source, "utf-8");
2306
- const bodyPath = await this.bodyStore.putMessage(accountId, folderId, uid, raw);
2307
- this.db.updateBodyPath(accountId, uid, bodyPath);
2308
- this.emit("bodyCached", accountId, uid);
2309
- counters.totalFetched++;
2310
- madeProgress = true;
2311
- }
2312
- catch (e) {
2313
- console.error(` [prefetch] ${accountId}/${uid}: store write failed: ${e.message}`);
2314
- }
2315
- })());
2316
- });
2317
- await Promise.all(pending);
2318
- }, { slow: true });
2319
- batchSucceeded = true;
2320
- this.clearFolderErrors(accountId, folder.path);
2321
- }
2322
- catch (e) {
2323
- const msg = String(e?.message || "");
2324
- console.error(` [prefetch] ${accountId} folder ${folder.path}: batch fetch failed: ${msg}`);
2325
- counters.errors++;
2326
- this.recordFolderError(accountId, folder.path);
2327
- if (counters.errors >= ERROR_BUDGET)
2328
- break;
2329
- }
2330
- // CRITICAL: only prune when the batch actually completed.
2331
- // A thrown batch means NOTHING was received; treating
2332
- // absence as server-deletion lost 296 messages once.
2333
- if (batchSucceeded)
2334
- for (const uid of uids) {
2335
- if (received.has(uid))
2336
- continue;
2337
- try {
2338
- this.unlinkBodyFile(accountId, uid, folderId).catch(() => { });
2339
- this.db.deleteMessage(accountId, uid, "prefetch batch: server didn't return body for queued UID — assumed deleted", "mailx-imap prefetchBodies (IMAP batch)");
2340
- counters.deleted++;
2341
- madeProgress = true;
2562
+ for (let chunkStart = 0; chunkStart < uids.length; chunkStart += PREFETCH_CHUNK_SIZE) {
2563
+ const chunk = uids.slice(chunkStart, chunkStart + PREFETCH_CHUNK_SIZE);
2564
+ const received = new Set();
2565
+ let batchSucceeded = false;
2566
+ try {
2567
+ // Slow lane: prefetch is the textbook "this
2568
+ // might take a while" case — let interactive
2569
+ // ops slip ahead. Each chunk is its own
2570
+ // withConnection so the queue drains the
2571
+ // fast lane between chunks instead of holding
2572
+ // the connection through the whole folder.
2573
+ await this.withConnection(accountId, async (client) => {
2574
+ const pending = [];
2575
+ await client.fetchBodiesBatch(folder.path, chunk, (uid, source) => {
2576
+ received.add(uid);
2577
+ pending.push((async () => {
2578
+ try {
2579
+ const raw = Buffer.from(source, "utf-8");
2580
+ const bodyPath = await this.bodyStore.putMessage(accountId, folderId, uid, raw);
2581
+ this.db.updateBodyPath(accountId, uid, bodyPath);
2582
+ this.emit("bodyCached", accountId, uid);
2583
+ counters.totalFetched++;
2584
+ madeProgress = true;
2585
+ }
2586
+ catch (e) {
2587
+ console.error(` [prefetch] ${accountId}/${uid}: store write failed: ${e.message}`);
2588
+ }
2589
+ })());
2590
+ });
2591
+ await Promise.all(pending);
2592
+ }, { slow: true });
2593
+ batchSucceeded = true;
2594
+ this.clearFolderErrors(accountId, folder.path);
2595
+ }
2596
+ catch (e) {
2597
+ const msg = String(e?.message || "");
2598
+ console.error(` [prefetch] ${accountId} folder ${folder.path} chunk ${chunkStart / PREFETCH_CHUNK_SIZE}: batch fetch failed: ${msg}`);
2599
+ counters.errors++;
2600
+ this.recordFolderError(accountId, folder.path);
2601
+ if (counters.errors >= ERROR_BUDGET)
2602
+ break;
2603
+ }
2604
+ // Prune missing UIDs only when the batch actually
2605
+ // completed AND returned at least one body. A
2606
+ // "successful but empty" batch (FETCH parser
2607
+ // missed every literal, wrong-folder selected,
2608
+ // mid-stream connection hiccup that didn't
2609
+ // throw) is indistinguishable from "all UIDs
2610
+ // were deleted server-side" without that signal
2611
+ // — and erring toward delete cost Bob ~66
2612
+ // messages on 2026-05-08 (`prefetch] bobma: 0
2613
+ // bodies cached, 66 stale rows pruned`). The
2614
+ // set-diff reconcile in syncFolder is the
2615
+ // authoritative deletion path with a 30-min
2616
+ // grace window; prefetch defers to it.
2617
+ const someReceived = received.size > 0;
2618
+ if (batchSucceeded && someReceived)
2619
+ for (const uid of chunk) {
2620
+ if (received.has(uid))
2621
+ continue;
2622
+ try {
2623
+ this.unlinkBodyFile(accountId, uid, folderId).catch(() => { });
2624
+ this.db.deleteMessage(accountId, uid, "prefetch batch: server didn't return body for queued UID — assumed deleted", "mailx-imap prefetchBodies (IMAP batch)");
2625
+ counters.deleted++;
2626
+ madeProgress = true;
2627
+ }
2628
+ catch { /* ignore */ }
2342
2629
  }
2343
- catch { /* ignore */ }
2630
+ else if (batchSucceeded && !someReceived) {
2631
+ console.error(` [prefetch] ${accountId}/${folder.path}: chunk ${chunkStart}-${chunkStart + chunk.length - 1} returned 0/${chunk.length} bodies — NOT pruning (set-diff reconcile owns deletion). UIDs: ${chunk.slice(0, 5).join(",")}${chunk.length > 5 ? "..." : ""}`);
2344
2632
  }
2633
+ }
2634
+ if (counters.errors >= ERROR_BUDGET)
2635
+ break;
2345
2636
  }
2346
2637
  if (counters.errors >= ERROR_BUDGET) {
2347
2638
  console.error(` [prefetch] ${accountId}: stopping after ${counters.errors} errors (${counters.totalFetched} cached, ${counters.deleted} pruned)`);
@@ -3029,6 +3320,20 @@ export class ImapManager extends EventEmitter {
3029
3320
  // retry. Foreign hosts are left alone — we have no way to know if their
3030
3321
  // process is alive. Cross-host stale recovery is the IMAP-folder path's
3031
3322
  // job (sweeper looks at server-side claim flags, not local files).
3323
+ // Stale-claim recovery. A claim is "stale" if any of:
3324
+ // (a) the PID is dead — original owner crashed mid-send
3325
+ // (b) the PID is alive BUT it's not us, and the file mtime is
3326
+ // older than STALE_CLAIM_MS — the OS recycled the PID for
3327
+ // some other process. `process.kill(pid, 0)` returning success
3328
+ // only proves *some* process owns that PID, not that it's
3329
+ // our long-dead mailx daemon. Without the age guard, a
3330
+ // claim survives forever as soon as any other Node process
3331
+ // (statusline, msger, npm) gets the recycled PID. Bob saw
3332
+ // this exact case: `.sending-rmf39-63196` sat in the queue
3333
+ // for 7+ hours because PID 63196 was now an unrelated Node.
3334
+ // (c) it's our PID — never sweep our own claim.
3335
+ const STALE_CLAIM_MS = 3600_000;
3336
+ const myPid = process.pid;
3032
3337
  for (const dir of [outboxDir, queuedDir]) {
3033
3338
  if (!fs.existsSync(dir))
3034
3339
  continue;
@@ -3040,17 +3345,28 @@ export class ImapManager extends EventEmitter {
3040
3345
  if (host !== this.hostname)
3041
3346
  continue;
3042
3347
  const pid = parseInt(pidStr);
3348
+ if (pid === myPid)
3349
+ continue; // it's us
3043
3350
  let alive = false;
3044
3351
  try {
3045
3352
  process.kill(pid, 0);
3046
3353
  alive = true;
3047
3354
  }
3048
3355
  catch { /* dead */ }
3049
- if (alive)
3050
- continue; // live claim — owner (sibling or self) still has it
3356
+ let ageMs = Infinity;
3357
+ try {
3358
+ ageMs = Date.now() - fs.statSync(path.join(dir, f)).mtimeMs;
3359
+ }
3360
+ catch { /* */ }
3361
+ // Live PID + recent mtime → assume genuine sibling owner.
3362
+ // Live PID + ancient mtime → PID got recycled, sweep it.
3363
+ // Dead PID → sweep regardless of age.
3364
+ if (alive && ageMs < STALE_CLAIM_MS)
3365
+ continue;
3051
3366
  try {
3052
3367
  fs.renameSync(path.join(dir, f), path.join(dir, original));
3053
- console.log(` [outbox] Recovered stale claim ${f} ${original}`);
3368
+ const reason = alive ? `recycled PID, mtime ${Math.round(ageMs / 60_000)}m old` : "dead PID";
3369
+ console.log(` [outbox] Recovered stale claim ${f} → ${original} (${reason})`);
3054
3370
  }
3055
3371
  catch { /* ignore */ }
3056
3372
  }
@@ -3351,15 +3667,33 @@ export class ImapManager extends EventEmitter {
3351
3667
  await client.deleteMessageByUid(outboxFolder.path, uid);
3352
3668
  }, { slow: true });
3353
3669
  if (sentFolder) {
3670
+ let appendedSentUid = null;
3354
3671
  try {
3355
3672
  await this.withConnection(accountId, async (client) => {
3356
- await client.appendMessage(sentFolder.path, source, ["\\Seen"]);
3673
+ appendedSentUid = await client.appendMessage(sentFolder.path, source, ["\\Seen"]);
3357
3674
  }, { slow: true });
3358
- this.syncFolder(accountId, sentFolder.id).catch(() => { });
3359
3675
  }
3360
3676
  catch (sentErr) {
3361
3677
  console.error(` [outbox] Failed to copy to Sent: ${sentErr.message} — message was sent successfully`);
3362
3678
  }
3679
+ if (appendedSentUid != null) {
3680
+ // The server's APPENDUID response gave us the exact UID
3681
+ // the message landed at in Sent. Insert the local row
3682
+ // directly from the source we already have — no IMAP
3683
+ // round-trip, no SELECT-then-FETCH dance. The Sent
3684
+ // folder view shows the message immediately. The next
3685
+ // periodic sync will see this UID already present and
3686
+ // no-op. Critical: this means a slow/stuck Sent SELECT
3687
+ // never blocks "show me what I just sent" — that was
3688
+ // the user-visible "where's my sent message?" bug.
3689
+ await this.insertLocalRowFromSource(accountId, sentFolder, appendedSentUid, source, ["\\Seen"])
3690
+ .catch((e) => console.error(` [outbox] Local Sent row insert failed: ${e?.message || e} — falling back to broad sync`));
3691
+ }
3692
+ else {
3693
+ // No APPENDUID — server doesn't support UIDPLUS, or
3694
+ // APPEND itself failed. Fall back to a broad sync.
3695
+ this.syncFolder(accountId, sentFolder.id).catch(() => { });
3696
+ }
3363
3697
  this.syncFolder(accountId, outboxFolder.id).catch(() => { });
3364
3698
  }
3365
3699
  }
@@ -3720,8 +4054,14 @@ export class ImapManager extends EventEmitter {
3720
4054
  this.stopPeriodicSync();
3721
4055
  this.stopOutboxWorker();
3722
4056
  await this.stopWatching();
3723
- // Disconnect all persistent operational connections
3724
- for (const [accountId] of this.opsClients) {
4057
+ // Disconnect persistent connections on both lanes. Use a Set
4058
+ // because fastClients can hold an entry an account doesn't have
4059
+ // in opsClients (e.g. body fetches happened but no sync did).
4060
+ const accountIds = new Set([
4061
+ ...this.opsClients.keys(),
4062
+ ...this.fastClients.keys(),
4063
+ ]);
4064
+ for (const accountId of accountIds) {
3725
4065
  await this.disconnectOps(accountId);
3726
4066
  }
3727
4067
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@bobfrankston/mailx-imap",
3
- "version": "0.1.28",
3
+ "version": "0.1.29",
4
4
  "type": "module",
5
5
  "main": "index.js",
6
6
  "types": "index.d.ts",
@@ -11,11 +11,11 @@
11
11
  "dependencies": {
12
12
  "@bobfrankston/mailx-types": "^0.1.10",
13
13
  "@bobfrankston/mailx-settings": "^0.1.13",
14
- "@bobfrankston/mailx-store": "^0.1.13",
15
- "@bobfrankston/iflow-direct": "^0.1.35",
16
- "@bobfrankston/tcp-transport": "^0.1.5",
17
- "@bobfrankston/smtp-direct": "^0.1.5",
18
- "@bobfrankston/mailx-sync": "^0.1.15",
14
+ "@bobfrankston/mailx-store": "^0.1.15",
15
+ "@bobfrankston/iflow-direct": "^0.1.39",
16
+ "@bobfrankston/tcp-transport": "^0.1.6",
17
+ "@bobfrankston/smtp-direct": "^0.1.6",
18
+ "@bobfrankston/mailx-sync": "^0.1.16",
19
19
  "@bobfrankston/oauthsupport": "^1.0.26"
20
20
  },
21
21
  "repository": {
@@ -39,11 +39,11 @@
39
39
  "dependencies": {
40
40
  "@bobfrankston/mailx-types": "^0.1.10",
41
41
  "@bobfrankston/mailx-settings": "^0.1.13",
42
- "@bobfrankston/mailx-store": "^0.1.13",
43
- "@bobfrankston/iflow-direct": "^0.1.35",
44
- "@bobfrankston/tcp-transport": "^0.1.5",
45
- "@bobfrankston/smtp-direct": "^0.1.5",
46
- "@bobfrankston/mailx-sync": "^0.1.15",
42
+ "@bobfrankston/mailx-store": "^0.1.15",
43
+ "@bobfrankston/iflow-direct": "^0.1.39",
44
+ "@bobfrankston/tcp-transport": "^0.1.6",
45
+ "@bobfrankston/smtp-direct": "^0.1.6",
46
+ "@bobfrankston/mailx-sync": "^0.1.16",
47
47
  "@bobfrankston/oauthsupport": "^1.0.26"
48
48
  }
49
49
  }