@bobfrankston/mailx 1.0.461 → 1.0.463

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (29) hide show
  1. package/client/lib/message-state.js +31 -12
  2. package/client/lib/message-state.js.map +1 -1
  3. package/client/lib/message-state.ts +31 -11
  4. package/package.json +5 -4
  5. package/packages/mailx-api/package.json +1 -1
  6. package/packages/mailx-core/package.json +1 -1
  7. package/packages/mailx-imap/.gitattributes +10 -0
  8. package/packages/mailx-imap/index.d.ts +442 -0
  9. package/packages/mailx-imap/index.d.ts.map +1 -0
  10. package/packages/mailx-imap/index.js +3684 -0
  11. package/packages/mailx-imap/index.js.map +1 -0
  12. package/packages/mailx-imap/index.ts +3652 -0
  13. package/packages/mailx-imap/package-lock.json +131 -0
  14. package/packages/mailx-imap/package.json +25 -0
  15. package/packages/mailx-imap/providers/gmail-api.d.ts +8 -0
  16. package/packages/mailx-imap/providers/gmail-api.d.ts.map +1 -0
  17. package/packages/mailx-imap/providers/gmail-api.js +8 -0
  18. package/packages/mailx-imap/providers/gmail-api.js.map +1 -0
  19. package/packages/mailx-imap/providers/gmail-api.ts +8 -0
  20. package/packages/mailx-imap/providers/outlook-api.ts +7 -0
  21. package/packages/mailx-imap/providers/types.d.ts +9 -0
  22. package/packages/mailx-imap/providers/types.d.ts.map +1 -0
  23. package/packages/mailx-imap/providers/types.js +9 -0
  24. package/packages/mailx-imap/providers/types.js.map +1 -0
  25. package/packages/mailx-imap/providers/types.ts +9 -0
  26. package/packages/mailx-imap/tsconfig.json +9 -0
  27. package/packages/mailx-imap/tsconfig.tsbuildinfo +1 -0
  28. package/packages/mailx-server/package.json +1 -1
  29. package/packages/mailx-service/package.json +1 -1
@@ -0,0 +1,3684 @@
1
+ /**
2
+ * @bobfrankston/mailx-imap
3
+ * Multi-account IMAP management wrapping iflow.
4
+ * Syncs messages to local store, emits events for new mail.
5
+ */
6
+ import { createAutoImapConfig, CompatImapClient } from "@bobfrankston/iflow-direct";
7
+ import { authenticateOAuth } from "@bobfrankston/oauthsupport";
8
+ import { FileMessageStore } from "@bobfrankston/mailx-store";
9
+ import { loadSettings, getStorePath, getConfigDir, getHistoryDays, getPrefetch } from "@bobfrankston/mailx-settings";
10
+ import { EventEmitter } from "node:events";
11
+ import * as fs from "node:fs";
12
+ import * as path from "node:path";
13
+ import { simpleParser } from "mailparser";
14
+ import { GmailApiProvider } from "./providers/gmail-api.js";
15
+ import { SmtpClient } from "@bobfrankston/smtp-direct";
16
+ import * as os from "node:os";
17
+ import { fileURLToPath } from "node:url";
18
+ // Well-known ports — no magic numbers
19
+ const SMTP_PORT_STARTTLS = 587;
20
+ const SMTP_PORT_IMPLICIT_TLS = 465;
21
+ /** Per-message SMTP retry delay: if a send attempt fails, wait this long before
22
+ * the same file is retried. Gives the server time to settle so a retry after a
23
+ * lost-ack doesn't arrive while the first copy is still being processed. */
24
+ const OUTBOX_RETRY_DELAY_MS = 60000;
25
+ /** Parse X-Mailx-Retry* tracking headers from a raw RFC822 message. */
26
+ function parseRetryInfo(raw) {
27
+ const headerEnd = raw.search(/\r?\n\r?\n/);
28
+ const headers = headerEnd >= 0 ? raw.slice(0, headerEnd) : raw;
29
+ const attemptCount = (headers.match(/^X-Mailx-Retry:/gmi) || []).length;
30
+ const nextMatch = headers.match(/^X-Mailx-Retry-After:\s*(.+)$/mi);
31
+ const parsed = nextMatch ? Date.parse(nextMatch[1].trim()) : NaN;
32
+ return { attemptCount, nextAttemptAt: Number.isFinite(parsed) ? parsed : 0 };
33
+ }
34
+ /** Remove every occurrence of a header field from a raw RFC822 message. */
35
+ function stripHeaderField(raw, name) {
36
+ const re = new RegExp(`^${name.replace(/[.*+?^${}()|[\\]\\\\]/g, "\\\\$&")}:.*\\r?\\n`, "gmi");
37
+ return raw.replace(re, "");
38
+ }
39
+ /** Insert a header line just before the header/body blank line. Preserves CRLF vs LF. */
40
+ function insertHeaderBeforeBody(raw, line) {
41
+ const m = raw.match(/\r?\n\r?\n/);
42
+ if (!m)
43
+ return raw + "\r\n" + line + "\r\n";
44
+ const nl = m[0].startsWith("\r\n") ? "\r\n" : "\n";
45
+ return raw.slice(0, m.index) + nl + line + raw.slice(m.index);
46
+ }
47
+ /** Error thrown when a message body can't be fetched because the server says
48
+ * the message is gone (deleted from another device, expunged, etc.). The
49
+ * caller uses this to remove the stale local row instead of showing a
50
+ * generic "fetch failed" error to the user. */
51
+ function makeNotFoundError(accountId, folderId, uid) {
52
+ const err = new Error(`Message ${accountId}/${folderId}/${uid} not found on server`);
53
+ err.isNotFound = true;
54
+ return err;
55
+ }
56
+ /** Extract full error detail with provenance */
57
+ function imapError(err) {
58
+ const msg = err.message || err.reason || err.code || (typeof err === "string" ? err : "");
59
+ const parts = [];
60
+ if (msg)
61
+ parts.push(msg);
62
+ if (err.responseText)
63
+ parts.push(err.responseText);
64
+ if (err.responseStatus)
65
+ parts.push(`[${err.responseStatus}]`);
66
+ if (err.code && err.code !== msg)
67
+ parts.push(`[${err.code}]`);
68
+ if (parts.length === 0)
69
+ parts.push(`Unexpected error: ${JSON.stringify(err).slice(0, 200)}`);
70
+ // Add source location if available
71
+ if (err.stack) {
72
+ const frame = err.stack.split("\n").find((l) => l.includes("imap") || l.includes("transport") || l.includes("compat"));
73
+ if (frame)
74
+ parts.push(`(${frame.trim().replace(/^at\s+/, "")})`);
75
+ }
76
+ return parts.join(" — ");
77
+ }
78
+ /** Convert iflow address objects to our EmailAddress */
79
+ function toEmailAddress(addr) {
80
+ return {
81
+ name: addr?.name || "",
82
+ address: addr?.address || ""
83
+ };
84
+ }
85
+ /** Convert array of iflow addresses */
86
+ function toEmailAddresses(addrs) {
87
+ if (!addrs)
88
+ return [];
89
+ return addrs.map(toEmailAddress);
90
+ }
91
+ /** Decode HTML entities (  & etc.) to plain characters */
92
+ function decodeEntities(text) {
93
+ return text
94
+ .replace(/&#(\d+);/g, (_, n) => String.fromCodePoint(Number(n)))
95
+ .replace(/&#x([0-9a-fA-F]+);/g, (_, h) => String.fromCodePoint(parseInt(h, 16)))
96
+ .replace(/&amp;/g, "&").replace(/&lt;/g, "<").replace(/&gt;/g, ">")
97
+ .replace(/&quot;/g, '"').replace(/&apos;/g, "'").replace(/&nbsp;/g, " ");
98
+ }
99
+ /** Extract a plain-text preview from message source */
100
+ async function extractPreview(source) {
101
+ try {
102
+ const parsed = await simpleParser(source);
103
+ const bodyText = parsed.text || "";
104
+ const bodyHtml = parsed.html || "";
105
+ // Use text part; fall back to stripping HTML tags if text is empty
106
+ let raw = bodyText || bodyHtml.replace(/<[^>]+>/g, " ");
107
+ const preview = decodeEntities(raw).replace(/\s+/g, " ").trim().slice(0, 200);
108
+ const hasAttachments = (parsed.attachments?.length || 0) > 0;
109
+ return { bodyHtml, bodyText, preview, hasAttachments };
110
+ }
111
+ catch {
112
+ return { bodyHtml: "", bodyText: "", preview: "", hasAttachments: false };
113
+ }
114
+ }
115
+ /** Race a promise against a timeout. On timeout, forcibly logout the client to prevent hanging. */
116
+ async function withTimeout(promise, ms, client, label) {
117
+ let timer;
118
+ const timeout = new Promise((_, reject) => {
119
+ timer = setTimeout(() => {
120
+ // Force-close the client to unblock the hanging promise
121
+ try {
122
+ client.logout?.();
123
+ }
124
+ catch { /* ignore */ }
125
+ reject(new Error(`${label} timeout (${ms / 1000}s)`));
126
+ }, ms);
127
+ });
128
+ try {
129
+ return await Promise.race([promise, timeout]);
130
+ }
131
+ finally {
132
+ clearTimeout(timer);
133
+ }
134
+ }
135
+ export class ImapManager extends EventEmitter {
136
+ configs = new Map();
137
+ watchers = new Map();
138
+ fetchClients = new Map();
139
+ db;
140
+ bodyStore;
141
+ syncIntervals = new Map();
142
+ /** Track which accounts have already shown an error banner — only emit once per session */
143
+ accountErrorShown = new Set();
144
+ syncing = false;
145
+ inboxSyncing = false;
146
+ /** Use native IMAP client instead of imapflow. Set to true to enable. */
147
+ useNativeClient = false;
148
+ // Connection management: see withConnection() below.
149
+ // Cap-hit backoff machinery removed — bounded per-account concurrency
150
+ // (one ops socket + one IDLE socket) keeps mailx well under any
151
+ // reasonable server cap, so the recovery timer was dead weight that
152
+ // mostly served to lock the UI for minutes after a transient failure.
153
+ /** Per-account health counters. Incremented when the server misbehaves
154
+ * in ways that suggest a problem the user should know about (inactivity
155
+ * timeouts, connection-cap hits, rate-limit waits). Surfaced via a
156
+ * `diagnostics` event + `getDiagnostics` IPC so the UI can show a ⚠
157
+ * badge instead of burying the issue in the log. */
158
+ diagnostics = new Map();
159
+ getDiagnosticsEntry(accountId) {
160
+ let d = this.diagnostics.get(accountId);
161
+ if (!d) {
162
+ d = { accountId, inactivityTimeouts: 0, connCapHits: 0, rateLimitWaits: 0, lastTimeoutAt: 0, lastCommand: "", lastError: "" };
163
+ this.diagnostics.set(accountId, d);
164
+ }
165
+ return d;
166
+ }
167
+ /** Classify an error message and bump the relevant counter; emit the
168
+ * updated diagnostics snapshot. Call this from every catch in the sync
169
+ * paths so the UI can count "something's wrong" in real time. */
170
+ recordError(accountId, errMsg) {
171
+ const d = this.getDiagnosticsEntry(accountId);
172
+ if (/inactivity timeout/i.test(errMsg)) {
173
+ d.inactivityTimeouts++;
174
+ d.lastTimeoutAt = Date.now();
175
+ const m = errMsg.match(/A\d+ [A-Z ]+.*$/);
176
+ if (m)
177
+ d.lastCommand = m[0].slice(0, 120);
178
+ }
179
+ else if (/UNAVAILABLE|Maximum number of connections|too many connections/i.test(errMsg)) {
180
+ d.connCapHits++;
181
+ }
182
+ else if (/429|rate limit/i.test(errMsg)) {
183
+ d.rateLimitWaits++;
184
+ }
185
+ else {
186
+ return; // not a known diagnostic class — don't emit
187
+ }
188
+ d.lastError = errMsg.slice(0, 200);
189
+ this.emit("diagnostics", accountId, { ...d });
190
+ }
191
+ /** Public read for the IPC surface: snapshot of all account diagnostics. */
192
+ getDiagnosticsSnapshot() {
193
+ return Array.from(this.diagnostics.values()).map(d => ({ ...d }));
194
+ }
195
+ transportFactory;
196
+ constructor(db, transportFactory) {
197
+ super();
198
+ this.db = db;
199
+ this.transportFactory = transportFactory;
200
+ const storePath = getStorePath();
201
+ this.bodyStore = new FileMessageStore(storePath);
202
+ }
203
+ /** Get OAuth access token for an account (for SMTP auth) */
204
+ async getOAuthToken(accountId) {
205
+ const config = this.configs.get(accountId);
206
+ if (!config || !config.tokenProvider)
207
+ return null;
208
+ return config.tokenProvider();
209
+ }
210
+ /** Accounts currently re-authenticating — all operations skip these */
211
+ reauthenticating = new Set();
212
+ /** Last reauth attempt timestamp per account — prevents reauth loops (5 min cooldown) */
213
+ lastReauthAttempt = new Map();
214
+ /** Force re-authentication for an OAuth account — deletes cached IMAP token, triggers browser consent */
215
+ async reauthenticate(accountId) {
216
+ if (this.reauthenticating.has(accountId))
217
+ return false; // already in progress
218
+ this.reauthenticating.add(accountId);
219
+ try {
220
+ const settings = loadSettings();
221
+ const account = settings.accounts.find(a => a.id === accountId);
222
+ if (!account)
223
+ return false;
224
+ // Stop IDLE watcher for this account
225
+ const stopWatcher = this.watchers.get(accountId);
226
+ if (stopWatcher) {
227
+ try {
228
+ await stopWatcher();
229
+ }
230
+ catch { /* */ }
231
+ this.watchers.delete(accountId);
232
+ }
233
+ // Delete only the IMAP token (not contacts — separate scope, separate consent)
234
+ const accountDir = account.imap.user.replace(/[@.]/g, "_");
235
+ const tokenDir = path.join(getConfigDir(), "tokens", accountDir);
236
+ const tokenPath = path.join(tokenDir, "token.json");
237
+ if (fs.existsSync(tokenPath)) {
238
+ fs.unlinkSync(tokenPath);
239
+ console.log(` [reauth] Deleted ${tokenPath}`);
240
+ }
241
+ // Re-register the account to get a fresh config with new tokenProvider
242
+ this.configs.delete(accountId);
243
+ await this.addAccount(account);
244
+ // addAccount already pre-validates the token (opens browser if needed)
245
+ const config = this.configs.get(accountId);
246
+ if (config?.tokenProvider) {
247
+ console.log(` [reauth] ${accountId}: success`);
248
+ this.accountErrorShown.delete(accountId);
249
+ this.syncInbox().catch(() => { });
250
+ return true;
251
+ }
252
+ }
253
+ catch (e) {
254
+ console.error(` [reauth] ${accountId}: ${e.message}`);
255
+ }
256
+ finally {
257
+ this.reauthenticating.delete(accountId);
258
+ }
259
+ return false;
260
+ }
261
+ /** Delete a message directly on the IMAP server (for stuck outbox messages not in local DB) */
262
+ async deleteOnServer(accountId, folderPath, uid) {
263
+ return this.withConnection(accountId, async (client) => {
264
+ await client.deleteMessageByUid(folderPath, uid);
265
+ console.log(` Deleted UID ${uid} from ${folderPath} on server`);
266
+ });
267
+ }
268
+ /** Search messages on the IMAP server — returns matching UIDs */
269
+ async searchOnServer(accountId, mailboxPath, criteria) {
270
+ return this.withConnection(accountId, async (client) => {
271
+ return await client.searchMessages(mailboxPath, criteria);
272
+ });
273
+ }
274
+ /** Server-side search that also materializes any UIDs we don't yet have
275
+ * locally. Returns the full result after upsert, so the caller can
276
+ * render hits that fall outside the history window. The fetch loop
277
+ * can be long for big hit-sets, so this runs on the slow lane and
278
+ * yields between chunks (each chunk is a separate withConnection)
279
+ * so an interactive body fetch can interleave. */
280
+ async searchAndFetchOnServer(accountId, folderId, mailboxPath, criteria) {
281
+ const uids = await this.withConnection(accountId, async (client) => {
282
+ return await client.searchMessages(mailboxPath, criteria);
283
+ });
284
+ if (uids.length === 0)
285
+ return [];
286
+ const have = new Set(this.db.getUidsForFolder(accountId, folderId));
287
+ const missing = uids.filter(u => !have.has(u));
288
+ if (missing.length > 0) {
289
+ const folder = this.db.getFolders(accountId).find(f => f.id === folderId);
290
+ if (folder) {
291
+ const CHUNK = 500;
292
+ for (let i = 0; i < missing.length; i += CHUNK) {
293
+ const range = missing.slice(i, i + CHUNK).join(",");
294
+ await this.withConnection(accountId, async (client) => {
295
+ const fetched = await client.fetchMessages(mailboxPath, range, { source: false });
296
+ if (fetched?.length) {
297
+ await this.storeMessages(accountId, folderId, folder, fetched, 0);
298
+ }
299
+ }, { slow: true });
300
+ }
301
+ this.db.recalcFolderCounts(folderId);
302
+ }
303
+ }
304
+ return uids;
305
+ }
306
+ /** Create a fresh IMAP client for an account (public access for API endpoints) */
307
+ async createPublicClient(accountId) {
308
+ return this.createClient(accountId);
309
+ }
310
+ // Legacy fallback disabled — was doubling connections without helping.
311
+ // To re-enable: uncomment legacyFallbacks logic in createClient and _syncAll.
312
+ // private legacyFallbacks = new Set<string>();
313
+ // ── Connection management: one persistent connection per account ──
314
+ // All operations on an account are serialized through an operation queue.
315
+ // No semaphore, no pool, no per-operation connect/disconnect.
316
+ // IDLE uses a separate connection (see startWatching).
317
+ /** Persistent operational connections — one per account, reused for all operations.
318
+ * Body fetch, sync, prefetch, outbox-append, flag/move all serialize through
319
+ * this single client per account via withConnection(). The priority lane
320
+ * in the queue lets interactive clicks jump ahead of background prefetch. */
321
+ opsClients = new Map();
322
+ /** Two-lane operation queue per account — interactive ops (body fetch on
323
+ * click, flag toggle) drain before background ops (sync, prefetch). FIFO
324
+ * within each lane. The single ops connection means there's never a race
325
+ * on which folder is SELECTed; commands run strictly sequentially. */
326
+ opsQueues = new Map();
327
+ /** Per-host semaphore — caps simultaneous IMAP socket opens to one server.
328
+ * Defensive guardrail: with the single-ops-per-account model an individual
329
+ * user's mailx never hits more than (#accounts × 2) sockets per host, well
330
+ * under any reasonable server cap. Exists for the multi-account-on-one-host
331
+ * case (e.g. bobma + bobma2 both on imap.iecc.com). */
332
+ hostSemaphores = new Map();
333
+ static HOST_PERMITS = 4;
334
+ /** Get (or create) the persistent operational connection for an account.
335
+ * logout() is wrapped as a no-op so legacy callers don't close it. */
336
+ async getOpsClient(accountId) {
337
+ let client = this.opsClients.get(accountId);
338
+ if (client) {
339
+ // C38: health-check the cached client before returning. If the
340
+ // underlying socket is dead (Dovecot silently dropped IDLE after
341
+ // the inactivity timeout, or we lost connectivity), the next
342
+ // command would fail with "Not connected" — and nothing would
343
+ // recover it until an explicit reconnectOps was called from the
344
+ // catch handler. Cheap pre-check here catches it earlier.
345
+ const sock = client?.native?.transport?.socket;
346
+ const dead = sock?.destroyed || sock?.readyState === "closed" || client?._dead;
347
+ if (!dead)
348
+ return client;
349
+ try {
350
+ await (client._realLogout || client.logout)();
351
+ }
352
+ catch { /* */ }
353
+ this.opsClients.delete(accountId);
354
+ console.log(` [conn] ${accountId}: stale ops client detected in getOpsClient — reconnecting`);
355
+ client = undefined;
356
+ }
357
+ client = await this.newClient(accountId, "ops");
358
+ // Wrap logout as no-op — this is a persistent connection. The
359
+ // newClient wrapper's close-counter runs on `_realLogout`.
360
+ const realLogout = client.logout.bind(client);
361
+ client.logout = async () => { };
362
+ client._realLogout = realLogout;
363
+ this.opsClients.set(accountId, client);
364
+ return client;
365
+ }
366
+ /** Run an operation on the account's connection — queued, sequential, no concurrency */
367
+ /** Run an operation against the account's single ops connection. Tasks
368
+ * queue strictly sequentially per account — only one IMAP command in
369
+ * flight at a time. This eliminates the SELECT-races and "stale client
370
+ * recovery" paths the old multi-client design needed.
371
+ *
372
+ * Default lane is `fast` — covers virtually everything (body fetch,
373
+ * flag toggle, move, incremental sync). Pass `slow: true` only for
374
+ * operations the caller knows will take a long time and shouldn't
375
+ * block the user (multi-folder prefetch batches, large backfills).
376
+ * When both lanes have tasks, fast drains first.
377
+ *
378
+ * Within a lane, FIFO. The running task always finishes — IMAP can't
379
+ * preempt a command mid-flight. */
380
+ async withConnection(accountId, fn, opts = {}) {
381
+ let queue = this.opsQueues.get(accountId);
382
+ if (!queue) {
383
+ queue = { fast: [], slow: [], running: false };
384
+ this.opsQueues.set(accountId, queue);
385
+ }
386
+ // Per-task wall-clock cap. Without one, a wedged IMAP command (TCP
387
+ // half-open, server stalled mid-FETCH) keeps the queue's running flag
388
+ // set forever and every subsequent fast-lane task — including the
389
+ // retry button the user just hit — waits behind it. Default is
390
+ // generous; callers driving user-visible reads pass a tighter value.
391
+ const timeoutMs = opts.timeoutMs ?? 90_000;
392
+ return new Promise((resolve, reject) => {
393
+ const task = async () => {
394
+ let timer;
395
+ try {
396
+ const client = await this.getOpsClient(accountId);
397
+ const result = await Promise.race([
398
+ fn(client),
399
+ new Promise((_, rej) => {
400
+ timer = setTimeout(() => rej(new Error(`ops timeout after ${Math.round(timeoutMs / 1000)}s — discarding client`)), timeoutMs);
401
+ }),
402
+ ]);
403
+ clearTimeout(timer);
404
+ resolve(result);
405
+ }
406
+ catch (e) {
407
+ clearTimeout(timer);
408
+ // Discard client on any error — keeping a half-broken
409
+ // socket poisoned every subsequent request. Destroy
410
+ // synchronously kills the in-flight command's socket so
411
+ // the underlying promise rejects and stops holding state.
412
+ const stale = this.opsClients.get(accountId);
413
+ this.opsClients.delete(accountId);
414
+ if (stale) {
415
+ try {
416
+ await (stale._realLogout || stale.logout)();
417
+ }
418
+ catch { /* */ }
419
+ try {
420
+ stale.destroy?.();
421
+ }
422
+ catch { /* */ }
423
+ }
424
+ reject(e);
425
+ }
426
+ };
427
+ (opts.slow ? queue.slow : queue.fast).push(task);
428
+ this.drainOpsQueue(accountId);
429
+ });
430
+ }
431
+ /** Run the next queued task. Fast lane drains before slow.
432
+ * Idempotent — safe to call after each task completes; the running
433
+ * flag prevents reentrant draining. */
434
+ drainOpsQueue(accountId) {
435
+ const queue = this.opsQueues.get(accountId);
436
+ if (!queue || queue.running)
437
+ return;
438
+ const next = queue.fast.shift() || queue.slow.shift();
439
+ if (!next)
440
+ return;
441
+ queue.running = true;
442
+ next().finally(() => {
443
+ queue.running = false;
444
+ this.drainOpsQueue(accountId);
445
+ });
446
+ }
447
+ /** Acquire one slot of the per-host connection semaphore. Returns a release
448
+ * function — call exactly once when the socket is closed. Used by
449
+ * newClient to cap simultaneous IMAP connections to a single server
450
+ * across all mailx accounts. */
451
+ acquireHostSlot(host) {
452
+ let sem = this.hostSemaphores.get(host);
453
+ if (!sem) {
454
+ sem = { permits: ImapManager.HOST_PERMITS, waiters: [] };
455
+ this.hostSemaphores.set(host, sem);
456
+ }
457
+ const semRef = sem;
458
+ return new Promise(resolve => {
459
+ const grant = () => {
460
+ semRef.permits--;
461
+ let released = false;
462
+ resolve(() => {
463
+ if (released)
464
+ return;
465
+ released = true;
466
+ semRef.permits++;
467
+ const next = semRef.waiters.shift();
468
+ if (next)
469
+ next();
470
+ });
471
+ };
472
+ if (semRef.permits > 0)
473
+ grant();
474
+ else
475
+ semRef.waiters.push(grant);
476
+ });
477
+ }
478
+ /** Open IMAP clients per account, used to trace who's opening sockets
479
+ * when we hit the Dovecot per-user+IP connection cap. */
480
+ openClients = new Map();
481
+ /** Create a new IMAP client (internal — callers use getOpsClient or withConnection).
482
+ * Acquires one slot of the per-host semaphore before constructing the
483
+ * client; the slot is released when logout() or destroy() runs.
484
+ * `purpose` is a short tag printed alongside the `[conn+]` log so we can
485
+ * tell which code path (ops/idle/etc.) opened each connection. */
486
+ async newClient(accountId, purpose = "?") {
487
+ if (this.reauthenticating.has(accountId))
488
+ throw new Error(`Account ${accountId} is re-authenticating`);
489
+ const config = this.configs.get(accountId);
490
+ if (!config)
491
+ throw new Error(`No config for account ${accountId}`);
492
+ const host = config.server || accountId;
493
+ const releaseHostSlot = await this.acquireHostSlot(host);
494
+ let client;
495
+ try {
496
+ client = new CompatImapClient(config, this.transportFactory);
497
+ }
498
+ catch (e) {
499
+ releaseHostSlot();
500
+ throw e;
501
+ }
502
+ let open = this.openClients.get(accountId);
503
+ if (!open) {
504
+ open = new Set();
505
+ this.openClients.set(accountId, open);
506
+ }
507
+ open.add(client);
508
+ console.log(` [conn+] ${accountId} (${purpose}) — ${open.size} open`);
509
+ let closed = false;
510
+ const markClosed = (how) => {
511
+ if (closed)
512
+ return;
513
+ closed = true;
514
+ open.delete(client);
515
+ releaseHostSlot();
516
+ console.log(` [conn-] ${accountId} (${purpose}/${how}) — ${open.size} open`);
517
+ };
518
+ const origLogout = client.logout?.bind(client);
519
+ if (origLogout) {
520
+ client.logout = async () => {
521
+ try {
522
+ await origLogout();
523
+ }
524
+ finally {
525
+ markClosed("logout");
526
+ }
527
+ };
528
+ }
529
+ const origDestroy = client.destroy?.bind(client);
530
+ if (origDestroy) {
531
+ client.destroy = () => {
532
+ try {
533
+ origDestroy();
534
+ }
535
+ finally {
536
+ markClosed("destroy");
537
+ }
538
+ };
539
+ }
540
+ return client;
541
+ }
542
+ /** Force-close every IMAP socket for an account — ops + any lingering
543
+ * ones in openClients (e.g. an IDLE watcher in flight). Used during
544
+ * account removal and disconnectOps so the server's connection slots
545
+ * free immediately rather than waiting for socket idle timeouts. */
546
+ async closeAllClients(accountId) {
547
+ const ops = this.opsClients.get(accountId);
548
+ this.opsClients.delete(accountId);
549
+ if (ops) {
550
+ try {
551
+ await (ops._realLogout || ops.logout)();
552
+ }
553
+ catch { /* */ }
554
+ try {
555
+ ops.destroy?.();
556
+ }
557
+ catch { /* */ }
558
+ }
559
+ const open = this.openClients.get(accountId);
560
+ if (open) {
561
+ for (const c of Array.from(open)) {
562
+ try {
563
+ await (c._realLogout || c.logout)?.();
564
+ }
565
+ catch { /* */ }
566
+ try {
567
+ c.destroy?.();
568
+ }
569
+ catch { /* */ }
570
+ }
571
+ open.clear();
572
+ }
573
+ }
574
+ /** Disconnect the persistent operational connection for an account */
575
+ async disconnectOps(accountId) {
576
+ const client = this.opsClients.get(accountId);
577
+ this.opsClients.delete(accountId);
578
+ if (client) {
579
+ // Force-close: don't wait for LOGOUT on a possibly dead socket
580
+ try {
581
+ const timeout = new Promise(r => setTimeout(r, 2000));
582
+ await Promise.race([(client._realLogout || client.logout)(), timeout]);
583
+ }
584
+ catch { /* */ }
585
+ // Destroy underlying socket if still open
586
+ try {
587
+ client.destroy?.();
588
+ }
589
+ catch { /* */ }
590
+ console.log(` [conn] ${accountId}: disconnected`);
591
+ }
592
+ }
593
+ /** Legacy entry: returns the shared persistent ops client. Most callers
594
+ * should be using `withConnection()` instead — that gives proper
595
+ * queueing and lets fast operations jump ahead of slow ones. */
596
+ async createClientWithLimit(accountId) {
597
+ return this.getOpsClient(accountId);
598
+ }
599
+ /** Disposable fresh client — only used by the IDLE watcher, which holds
600
+ * its own socket so the fast/slow ops queue isn't blocked by IDLE
601
+ * parking the connection in a wait-for-server state. */
602
+ async createClient(accountId, purpose = "misc") {
603
+ return this.newClient(accountId, purpose);
604
+ }
605
+ trackLogout(_accountId) { }
606
+ /** Number of registered IMAP accounts */
607
+ getAccountCount() { return this.configs.size; }
608
+ /** Register an account */
609
+ async addAccount(account) {
610
+ if (this.configs.has(account.id))
611
+ return;
612
+ // createAutoImapConfig auto-detects Gmail from server/username and sets up OAuth
613
+ // For OAuth accounts, provide a tokenProvider using oauthsupport
614
+ let tokenProvider;
615
+ if (account.imap.auth === "oauth2" || (!account.imap.password && account.imap.host?.includes("gmail"))) {
616
+ // Find Google OAuth credentials — check ~/.mailx first, then iflow-direct package
617
+ let credPath = path.join(getConfigDir(), "google-credentials.json");
618
+ if (!fs.existsSync(credPath)) {
619
+ try {
620
+ // Use fileURLToPath, NOT string-replace on "file://" — on Linux,
621
+ // file:///usr/local/... loses its leading slash via .replace("file:///",
622
+ // "") and becomes relative, so fs.existsSync silently fails.
623
+ const pkgDir = path.dirname(fileURLToPath(import.meta.resolve("@bobfrankston/iflow-direct")));
624
+ for (const name of ["iflow-credentials.json"]) {
625
+ const p = path.join(pkgDir, name);
626
+ if (fs.existsSync(p)) {
627
+ credPath = p;
628
+ break;
629
+ }
630
+ }
631
+ }
632
+ catch { /* iflow-direct not resolvable */ }
633
+ }
634
+ const tokenDir = path.join(getConfigDir(), "tokens", account.imap.user.replace(/[@.]/g, "_"));
635
+ tokenProvider = async () => {
636
+ // Wall-clock timeout on OAuth. Use a longer timeout when no
637
+ // cached token exists (first auth needs user to click through
638
+ // the browser consent screen — 30s is too tight).
639
+ const hasToken = fs.existsSync(path.join(tokenDir, "oauth-token.json"));
640
+ const TOKEN_FETCH_TIMEOUT_MS = hasToken ? 30_000 : 120_000;
641
+ const authPromise = authenticateOAuth(credPath, {
642
+ // Scope set covers two-way sync of all mailx-managed local
643
+ // stores: mail (mail.google.com), contacts (full, not
644
+ // readonly — we write edits back), calendar (full), tasks
645
+ // (full), drive (for shared accounts.jsonc).
646
+ scope: "https://mail.google.com/ https://www.googleapis.com/auth/contacts https://www.googleapis.com/auth/calendar https://www.googleapis.com/auth/tasks https://www.googleapis.com/auth/drive",
647
+ tokenDirectory: tokenDir,
648
+ credentialsKey: "installed",
649
+ loginHint: account.imap.user,
650
+ });
651
+ const timeoutPromise = new Promise((_, reject) => setTimeout(() => reject(new Error(`OAuth token fetch timeout (${TOKEN_FETCH_TIMEOUT_MS / 1000}s)`)), TOKEN_FETCH_TIMEOUT_MS));
652
+ const result = await Promise.race([authPromise, timeoutPromise]);
653
+ return result?.access_token || "";
654
+ };
655
+ }
656
+ // Non-Gmail accounts (typically Dovecot / generic IMAP) get a smaller
657
+ // fetch chunk size (10 vs 25) and longer inactivity timeout (300s) so
658
+ // multi-body FETCH batches don't trip the connection-dead detector on
659
+ // slow servers. Gmail stays at the defaults since it's fast and has
660
+ // its own rate limits to respect.
661
+ const isGmail = account.imap.host?.includes("gmail") || account.email.endsWith("@gmail.com");
662
+ const config = createAutoImapConfig({
663
+ server: account.imap.host,
664
+ port: account.imap.port,
665
+ username: account.imap.user,
666
+ password: account.imap.password,
667
+ tokenProvider,
668
+ inactivityTimeout: isGmail ? 60000 : 300000,
669
+ fetchChunkSize: isGmail ? 25 : 10,
670
+ fetchChunkSizeMax: isGmail ? 500 : 100,
671
+ });
672
+ this.configs.set(account.id, config);
673
+ // Register account in DB
674
+ this.db.upsertAccount(account.id, account.name, account.email, JSON.stringify(account));
675
+ // Pre-validate OAuth token (so browser consent happens now, not during a timed sync)
676
+ if (config.tokenProvider) {
677
+ try {
678
+ await config.tokenProvider();
679
+ console.log(` [auth] ${account.id}: token valid`);
680
+ }
681
+ catch (e) {
682
+ const errMsg = imapError(e);
683
+ console.error(` [auth] ${account.id}: ${errMsg}`);
684
+ const isTransient = /timeout|ECONNREFUSED|ECONNRESET|ETIMEDOUT|ENETUNREACH/i.test(errMsg);
685
+ if (isTransient) {
686
+ console.log(` [transient] ${account.id}: ${errMsg} — will retry on first sync`);
687
+ }
688
+ else if (!this.accountErrorShown.has(account.id)) {
689
+ this.accountErrorShown.add(account.id);
690
+ const config = this.configs.get(account.id);
691
+ this.emit("accountError", account.id, errMsg, errMsg, !!config?.tokenProvider);
692
+ }
693
+ }
694
+ }
695
+ }
696
+ /** Check if an account uses Gmail (should use API instead of IMAP) */
697
+ isGmailAccount(accountId) {
698
+ const settings = loadSettings();
699
+ const account = settings.accounts.find(a => a.id === accountId);
700
+ if (!account)
701
+ return false;
702
+ return account.imap?.host?.includes("gmail") || account.email?.endsWith("@gmail.com") || false;
703
+ }
704
+ /** Get a Gmail API provider for an account (reuses tokenProvider from IMAP config) */
705
+ getGmailProvider(accountId) {
706
+ const config = this.configs.get(accountId);
707
+ if (!config?.tokenProvider)
708
+ throw new Error(`No tokenProvider for ${accountId}`);
709
+ return new GmailApiProvider(config.tokenProvider);
710
+ }
711
+ /** Convert ProviderMessage to the shape expected by storeMessages/upsertMessage */
712
+ providerMsgToLocal(msg) {
713
+ return {
714
+ uid: msg.uid,
715
+ messageId: msg.messageId,
716
+ date: msg.date || new Date(),
717
+ subject: msg.subject,
718
+ from: msg.from,
719
+ to: msg.to,
720
+ cc: msg.cc,
721
+ seen: msg.seen,
722
+ flagged: msg.flagged,
723
+ answered: msg.answered,
724
+ draft: msg.draft,
725
+ size: msg.size,
726
+ source: msg.source,
727
+ providerId: msg.providerId,
728
+ };
729
+ }
730
+ /** Sync folder list for an account */
731
+ async syncFolders(accountId, client) {
732
+ if (!client)
733
+ client = await this.getOpsClient(accountId);
734
+ this.emit("syncProgress", accountId, "folders", 0);
735
+ const t0 = Date.now();
736
+ console.log(` [diag] ${accountId}: getFolderList starting...`);
737
+ const folders = await client.getFolderList();
738
+ console.log(` [diag] ${accountId}: getFolderList done in ${Date.now() - t0}ms (${folders.length} folders)`);
739
+ const specialFolders = client.getSpecialFolders(folders);
740
+ // Collect server paths so we can prune anything the server no longer
741
+ // has (user-renamed / -deleted / case-flipped a folder from another
742
+ // client). IMAP paths are case-sensitive, so "Foo" → "foo" is a real
743
+ // delete+create of two distinct mailboxes.
744
+ const serverPaths = new Set();
745
+ for (const folder of folders) {
746
+ // Skip non-selectable folders (virtual parents like "Added", "Added2")
747
+ const flags = folder.flags;
748
+ const flagArr = flags instanceof Set ? [...flags] : (flags || []);
749
+ if (flagArr.some((f) => f.toLowerCase() === "\\noselect" || f.toLowerCase() === "\\nonexistent"))
750
+ continue;
751
+ let specialUse = null;
752
+ if (specialFolders.inbox === folder.path)
753
+ specialUse = "inbox";
754
+ else if (specialFolders.sent === folder.path)
755
+ specialUse = "sent";
756
+ else if (specialFolders.trash === folder.path)
757
+ specialUse = "trash";
758
+ else if (specialFolders.drafts === folder.path)
759
+ specialUse = "drafts";
760
+ else if (specialFolders.spam === folder.path || specialFolders.junk === folder.path)
761
+ specialUse = "junk";
762
+ else if (specialFolders.archive === folder.path)
763
+ specialUse = "archive";
764
+ this.db.upsertFolder(accountId, folder.path, folder.name || folder.path.split(folder.delimiter || "/").pop() || folder.path, specialUse, folder.delimiter || "/");
765
+ serverPaths.add(folder.path);
766
+ }
767
+ // Prune: any local folder whose exact path (case-sensitive) isn't in
768
+ // the server's list has been deleted or renamed server-side. Safety
769
+ // rails: only prune when the server returned a non-empty list (empty
770
+ // result is more likely a transient protocol / auth error than "all
771
+ // your folders were deleted"). Never prune INBOX under any
772
+ // circumstances — even a broken server response shouldn't make us
773
+ // drop the account's primary mailbox. All other special-use folders
774
+ // ARE prunable: if the user actually deleted Sent on the server,
775
+ // we should reflect that locally, and the next sync will re-detect
776
+ // the server's real Sent folder and re-upsert.
777
+ if (folders.length > 0) {
778
+ const localFolders = this.db.getFolders(accountId);
779
+ const stale = localFolders.filter(f => !serverPaths.has(f.path) &&
780
+ f.specialUse !== "inbox");
781
+ for (const f of stale) {
782
+ console.log(` [sync] ${accountId}: pruning stale folder "${f.path}" (id=${f.id}) — no longer on server`);
783
+ try {
784
+ this.db.deleteFolder(f.id);
785
+ }
786
+ catch (e) {
787
+ console.error(` [sync] ${accountId}: prune failed for "${f.path}": ${e.message}`);
788
+ }
789
+ }
790
+ if (stale.length > 0) {
791
+ this.emit("folderCountsChanged", accountId, {});
792
+ }
793
+ }
794
+ this.emit("syncProgress", accountId, "folders", 100);
795
+ // Notify UI that folder structure changed — triggers tree re-render
796
+ const dbFolders = this.db.getFolders(accountId);
797
+ this.emit("folderCountsChanged", accountId, {});
798
+ return dbFolders;
799
+ }
800
+ /** Store a batch of messages to DB immediately — used by onChunk for incremental sync */
801
+ async storeMessages(accountId, folderId, folder, msgs, highestUid) {
802
+ let stored = 0;
803
+ this.db.beginTransaction();
804
+ try {
805
+ for (const msg of msgs) {
806
+ // Debug: log subjects with non-ASCII to trace encoding issues
807
+ if (msg.subject && /[^\x00-\x7F]/.test(msg.subject)) {
808
+ const hex = Buffer.from(msg.subject, "utf-8").subarray(0, 40).toString("hex");
809
+ console.log(` [encoding] subject: "${msg.subject.substring(0, 60)}" hex: ${hex}`);
810
+ }
811
+ if (msg.uid <= highestUid)
812
+ continue; // already have it
813
+ // Tombstone check: if the user locally deleted this Message-ID,
814
+ // don't re-import it. Server-side EXPUNGE may lag, or reconcile
815
+ // may find the message in an old list snapshot. Without this,
816
+ // deleted messages reappear on the next sync pass.
817
+ if (msg.messageId && this.db.hasTombstone(accountId, msg.messageId)) {
818
+ console.log(` [tombstone] ${accountId}: skipping ${msg.messageId} (locally deleted)`);
819
+ continue;
820
+ }
821
+ const source = msg.source || "";
822
+ let bodyPath = "";
823
+ let preview = "";
824
+ let hasAttachments = false;
825
+ if (source) {
826
+ bodyPath = await this.bodyStore.putMessage(accountId, folderId, msg.uid, Buffer.from(source, "utf-8"));
827
+ const parsed = await extractPreview(source);
828
+ preview = parsed.preview;
829
+ hasAttachments = parsed.hasAttachments;
830
+ }
831
+ const flags = [];
832
+ if (msg.seen)
833
+ flags.push("\\Seen");
834
+ if (msg.flagged)
835
+ flags.push("\\Flagged");
836
+ if (msg.answered)
837
+ flags.push("\\Answered");
838
+ if (msg.draft)
839
+ flags.push("\\Draft");
840
+ this.db.upsertMessage({
841
+ accountId, folderId, uid: msg.uid,
842
+ messageId: msg.messageId || "",
843
+ inReplyTo: msg.inReplyTo || "",
844
+ references: [],
845
+ date: msg.date instanceof Date ? msg.date.getTime() : (typeof msg.date === "number" ? msg.date : Date.now()),
846
+ subject: msg.subject || "",
847
+ from: toEmailAddress(msg.from?.[0] || {}),
848
+ to: toEmailAddresses(msg.to || []),
849
+ cc: toEmailAddresses(msg.cc || []),
850
+ flags, size: msg.size || 0, hasAttachments, preview, bodyPath
851
+ });
852
+ stored++;
853
+ }
854
+ this.db.commitTransaction();
855
+ }
856
+ catch (e) {
857
+ this.db.rollbackTransaction();
858
+ console.error(` storeMessages error: ${e.message}`);
859
+ }
860
+ return stored;
861
+ }
862
+ /** Sync messages for a specific folder */
863
+ async syncFolder(accountId, folderId, client) {
864
+ if (!client)
865
+ client = await this.getOpsClient(accountId);
866
+ const prefetch = getPrefetch();
867
+ const folders = this.db.getFolders(accountId);
868
+ const folder = folders.find(f => f.id === folderId);
869
+ if (!folder)
870
+ throw new Error(`Folder ${folderId} not found`);
871
+ this.emit("syncProgress", accountId, `sync:${folder.path}`, 0);
872
+ // Get the highest UID we already have for this folder
873
+ const highestUid = this.db.getHighestUid(accountId, folderId);
874
+ console.log(` [sync] ${accountId}/${folder.path}: highestUid=${highestUid}, fetching...`);
875
+ let messages;
876
+ const firstSync = highestUid === 0;
877
+ const historyDays = getHistoryDays(accountId);
878
+ // historyDays=0 means "all". On first sync we still cap at 30 days
879
+ // so the UI isn't empty for minutes while SEARCH SINCE 1970 runs
880
+ // through a years-old mailbox. Once we have any local messages, the
881
+ // backfill below extends the window in 90-day chunks per sync cycle.
882
+ let effectiveDays = historyDays;
883
+ if (effectiveDays === 0 && firstSync)
884
+ effectiveDays = 30;
885
+ const startDate = effectiveDays > 0
886
+ ? new Date(Date.now() - effectiveDays * 86400000)
887
+ : new Date(0);
888
+ if (highestUid > 0) {
889
+ // Incremental: fetch new messages — prefetch bodies for offline access
890
+ const fetched = await client.fetchMessagesSinceUid(folder.path, highestUid, { source: false });
891
+ // Filter out the last known message (IMAP * always returns at least one)
892
+ messages = fetched.filter((m) => m.uid > highestUid);
893
+ // Gap detection: check for missing UIDs within the range we've already synced
894
+ // Only reconcile between our lowest and highest UID — don't try to fetch the entire folder history
895
+ const existingUids = this.db.getUidsForFolder(accountId, folderId);
896
+ if (existingUids.length > 0) {
897
+ try {
898
+ const lowestUid = Math.min(...existingUids);
899
+ // Fetch UIDs in our known range from IMAP
900
+ const rangeUids = await client.getUids(folder.path);
901
+ const rangeInScope = rangeUids.filter((uid) => uid >= lowestUid && uid <= highestUid);
902
+ const existingSet = new Set(existingUids);
903
+ const newSet = new Set(messages.map(m => m.uid));
904
+ const missingUids = rangeInScope.filter((uid) => !existingSet.has(uid) && !newSet.has(uid));
905
+ if (missingUids.length > 0 && missingUids.length <= 5000) {
906
+ console.log(` ${folder.path}: gap detected — ${missingUids.length} missing UIDs in range ${lowestUid}..${highestUid}`);
907
+ const chunkSize = 500;
908
+ let recoveredTotal = 0;
909
+ for (let i = 0; i < missingUids.length; i += chunkSize) {
910
+ const chunk = missingUids.slice(i, i + chunkSize);
911
+ const range = chunk.join(",");
912
+ const recovered = await client.fetchMessages(folder.path, range, { source: false });
913
+ messages.push(...recovered);
914
+ recoveredTotal += recovered.length;
915
+ console.log(` ${folder.path}: gap-fill ${recoveredTotal}/${missingUids.length}`);
916
+ }
917
+ }
918
+ else if (missingUids.length > 5000) {
919
+ console.log(` ${folder.path}: ${missingUids.length} missing UIDs — too many, skipping reconciliation (delete DB to force full re-sync)`);
920
+ }
921
+ }
922
+ catch (e) {
923
+ console.error(` ${folder.path}: gap detection failed: ${e.message}`);
924
+ }
925
+ }
926
+ // Backfill: if the history window reaches further back than our
927
+ // oldest local message, fetch the gap. Chunk 90 days per sync
928
+ // cycle so historyDays=0 catches up incrementally instead of
929
+ // asking Dovecot for SEARCH SINCE 1970 in one go.
930
+ const oldestDate = this.db.getOldestDate(accountId, folderId);
931
+ if (oldestDate > 0 && startDate.getTime() < oldestDate) {
932
+ try {
933
+ const CHUNK_MS = 90 * 86400000;
934
+ const chunkStart = new Date(Math.max(startDate.getTime(), oldestDate - CHUNK_MS));
935
+ const existingUids = new Set(this.db.getUidsForFolder(accountId, folderId));
936
+ const backfill = await client.fetchMessageByDate(folder.path, chunkStart, new Date(oldestDate), { source: false });
937
+ const newBackfill = backfill.filter((m) => !existingUids.has(m.uid));
938
+ if (newBackfill.length > 0) {
939
+ console.log(` ${folder.path}: backfilling ${newBackfill.length} older messages (${chunkStart.toISOString().slice(0, 10)} → ${new Date(oldestDate).toISOString().slice(0, 10)})`);
940
+ messages.push(...newBackfill);
941
+ }
942
+ }
943
+ catch (e) {
944
+ console.error(` ${folder.path}: backfill failed: ${e.message}`);
945
+ }
946
+ }
947
+ }
948
+ else {
949
+ // First sync: fetch in chunks, store each chunk immediately for instant UI
950
+ let totalStored = 0;
951
+ const onChunk = async (chunk) => {
952
+ const stored = await this.storeMessages(accountId, folderId, folder, chunk, highestUid);
953
+ totalStored += stored;
954
+ if (stored > 0) {
955
+ this.db.recalcFolderCounts(folderId);
956
+ this.emit("folderCountsChanged", accountId, {});
957
+ }
958
+ };
959
+ const tomorrow = new Date(Date.now() + 86400000); // IMAP BEFORE is exclusive
960
+ // First sync: metadata only for fast UI — bodies prefetched in background after
961
+ messages = await client.fetchMessageByDate(folder.path, startDate, tomorrow, { source: false }, onChunk);
962
+ if (totalStored > 0) {
963
+ console.log(` ${folder.path}: ${totalStored} messages (streamed)`);
964
+ this.db.recalcFolderCounts(folderId);
965
+ this.emit("folderCountsChanged", accountId, {});
966
+ this.emit("syncProgress", accountId, `sync:${folder.path}`, 100);
967
+ return totalStored;
968
+ }
969
+ }
970
+ if (messages.length > 0)
971
+ console.log(` ${folder.path}: ${messages.length} new messages`);
972
+ let newCount = 0;
973
+ const batchSize = 50;
974
+ for (let batchStart = 0; batchStart < messages.length; batchStart += batchSize) {
975
+ const batchEnd = Math.min(batchStart + batchSize, messages.length);
976
+ this.db.beginTransaction();
977
+ try {
978
+ for (let i = batchStart; i < batchEnd; i++) {
979
+ const msg = messages[i];
980
+ // Skip if we already have this UID
981
+ if (msg.uid <= highestUid) {
982
+ // But update flags in case they changed
983
+ const flags = [];
984
+ if (msg.seen)
985
+ flags.push("\\Seen");
986
+ if (msg.flagged)
987
+ flags.push("\\Flagged");
988
+ if (msg.answered)
989
+ flags.push("\\Answered");
990
+ if (msg.draft)
991
+ flags.push("\\Draft");
992
+ this.db.updateMessageFlags(accountId, msg.uid, flags);
993
+ continue;
994
+ }
995
+ // Tombstone check — same reason as the streamy onChunk path
996
+ // at storeMessages: a locally-deleted message that the server
997
+ // hasn't EXPUNGEd yet would otherwise reappear on next sync.
998
+ // User-visible symptom: "I deleted it but it came back."
999
+ if (msg.messageId && this.db.hasTombstone(accountId, msg.messageId)) {
1000
+ console.log(` [tombstone] ${accountId}: skipping ${msg.messageId} in syncFolder (locally deleted)`);
1001
+ continue;
1002
+ }
1003
+ // Store body
1004
+ const source = msg.source || "";
1005
+ let bodyPath = "";
1006
+ if (source) {
1007
+ bodyPath = await this.bodyStore.putMessage(accountId, folderId, msg.uid, Buffer.from(source, "utf-8"));
1008
+ }
1009
+ // Parse for preview and attachment info
1010
+ const parsed = await extractPreview(source);
1011
+ // Build flags array
1012
+ const flags = [];
1013
+ if (msg.seen)
1014
+ flags.push("\\Seen");
1015
+ if (msg.flagged)
1016
+ flags.push("\\Flagged");
1017
+ if (msg.answered)
1018
+ flags.push("\\Answered");
1019
+ if (msg.draft)
1020
+ flags.push("\\Draft");
1021
+ // Store metadata
1022
+ this.db.upsertMessage({
1023
+ accountId,
1024
+ folderId,
1025
+ uid: msg.uid,
1026
+ messageId: msg.messageId || "",
1027
+ inReplyTo: msg.inReplyTo || "",
1028
+ references: [],
1029
+ date: msg.date instanceof Date ? msg.date.getTime() : (typeof msg.date === "number" ? msg.date : Date.now()),
1030
+ subject: msg.subject || "",
1031
+ from: toEmailAddress(msg.from?.[0] || {}),
1032
+ to: toEmailAddresses(msg.to || []),
1033
+ cc: toEmailAddresses(msg.cc || []),
1034
+ flags,
1035
+ size: msg.size || 0,
1036
+ hasAttachments: parsed.hasAttachments,
1037
+ preview: parsed.preview,
1038
+ bodyPath
1039
+ });
1040
+ newCount++;
1041
+ }
1042
+ this.db.commitTransaction();
1043
+ }
1044
+ catch (e) {
1045
+ console.error(` transaction error: ${e.message}`);
1046
+ this.db.rollbackTransaction();
1047
+ throw e;
1048
+ }
1049
+ // Emit progress and notify client after each batch
1050
+ this.emit("syncProgress", accountId, `sync:${folder.path}`, Math.round((batchEnd / messages.length) * 100));
1051
+ // On first sync, emit folderCountsChanged per batch so newest messages appear immediately
1052
+ if (firstSync && newCount > 0) {
1053
+ this.db.recalcFolderCounts(folderId);
1054
+ const folderInfo = this.db.getFolders(accountId).find(f => f.id === folderId);
1055
+ this.emit("folderCountsChanged", accountId, {
1056
+ [folderId]: { total: folderInfo?.totalCount || 0, unread: folderInfo?.unreadCount || 0 }
1057
+ });
1058
+ }
1059
+ }
1060
+ if (newCount > 0)
1061
+ console.log(` stored ${newCount} new messages`);
1062
+ // Remove messages deleted on the server (skip on first sync — nothing to reconcile).
1063
+ //
1064
+ // SAFETY (same three guards the Gmail API path uses, see ~line 1388):
1065
+ // 1. Skip if server returned an empty list but we have local messages
1066
+ // (transient Dovecot error / connection hiccup returning empty UID SEARCH
1067
+ // must not wipe the folder).
1068
+ // 2. Refuse to delete more than 50% in one pass — indicates a sync bug,
1069
+ // never a real user action. User can fix with `mailx -rebuild` if real.
1070
+ // 3. Log every deletion with Message-ID + subject so future reports have
1071
+ // data (the "ubiquiti letter disappeared after reply" case had no trace).
1072
+ let deletedCount = 0;
1073
+ if (!firstSync) {
1074
+ try {
1075
+ const serverUidsArr = await client.getUids(folder.path);
1076
+ const serverUids = new Set(serverUidsArr);
1077
+ const localUids = this.db.getUidsForFolder(accountId, folderId);
1078
+ const toDelete = localUids.filter(uid => !serverUids.has(uid));
1079
+ if (serverUidsArr.length === 0 && localUids.length > 0) {
1080
+ console.log(` [sync] ${accountId}/${folder.path}: reconcile skipped — server UID list empty but local has ${localUids.length} (treating as transient)`);
1081
+ }
1082
+ else if (localUids.length > 0 && toDelete.length / localUids.length > 0.5) {
1083
+ console.log(` [sync] ${accountId}/${folder.path}: reconcile REFUSED — would delete ${toDelete.length}/${localUids.length} (${Math.round(toDelete.length / localUids.length * 100)}%) — probably a sync bug, skipping`);
1084
+ }
1085
+ else {
1086
+ for (const uid of toDelete) {
1087
+ const env = this.db.getMessageByUid(accountId, uid);
1088
+ const tag = env ? `msgid=${env.messageId || "?"} subj="${(env.subject || "").slice(0, 60)}"` : "unknown";
1089
+ console.log(` [reconcile-delete] ${accountId}/${folder.path} uid=${uid} ${tag}`);
1090
+ this.unlinkBodyFile(accountId, uid, folderId).catch(() => { });
1091
+ this.db.deleteMessage(accountId, uid);
1092
+ deletedCount++;
1093
+ }
1094
+ if (deletedCount > 0)
1095
+ console.log(` removed ${deletedCount} deleted messages`);
1096
+ }
1097
+ }
1098
+ catch (e) {
1099
+ console.error(` deletion sync error: ${e.message}`);
1100
+ }
1101
+ }
1102
+ // Update folder counts from local DB (after deletions + additions)
1103
+ // Use recalcFolderCounts — single SQL query instead of fetching all messages
1104
+ this.db.recalcFolderCounts(folderId);
1105
+ this.emit("syncProgress", accountId, `sync:${folder.path}`, 100);
1106
+ const syncedAt = Date.now();
1107
+ // Notify client to refresh if anything changed
1108
+ if (newCount > 0 || deletedCount > 0) {
1109
+ const updatedFolder = this.db.getFolders(accountId).find(f => f.id === folderId);
1110
+ this.emit("folderCountsChanged", accountId, {
1111
+ [folderId]: { total: updatedFolder?.totalCount || 0, unread: updatedFolder?.unreadCount || 0 }
1112
+ });
1113
+ }
1114
+ this.emit("folderSynced", accountId, folderId, syncedAt);
1115
+ this.db.updateLastSync(accountId, syncedAt);
1116
+ return newCount;
1117
+ }
1118
+ /** Sync all folders for all accounts */
1119
+ async syncAll() {
1120
+ if (this.syncing)
1121
+ return; // Prevent concurrent syncs
1122
+ this.syncing = true;
1123
+ try {
1124
+ await Promise.race([
1125
+ this._syncAll(),
1126
+ new Promise((_, reject) => setTimeout(() => reject(new Error("Global sync timeout (10min)")), 600000))
1127
+ ]);
1128
+ }
1129
+ catch (e) {
1130
+ console.error(` syncAll error: ${e.message}`);
1131
+ }
1132
+ finally {
1133
+ this.syncing = false;
1134
+ }
1135
+ }
1136
+ async _syncAll() {
1137
+ const priorityOrder = ["sent", "drafts", "archive", "junk", "trash"];
1138
+ // Sync all accounts in parallel — each manages its own connection.
1139
+ // Prefetch runs per-account immediately after that account's sync
1140
+ // completes, NOT after all accounts finish. This way a slow account
1141
+ // (bobma with 300s timeouts) doesn't block prefetch for a fast account
1142
+ // (Gmail). The old code put prefetch after `allSettled`, but syncAll
1143
+ // has a 10-minute wall-clock timeout that killed it first — so
1144
+ // prefetch never ran.
1145
+ const syncAndPrefetch = async (accountId) => {
1146
+ try {
1147
+ await this.syncAccount(accountId, priorityOrder);
1148
+ }
1149
+ catch {
1150
+ // syncAccount already logs + emits syncError. Don't follow a
1151
+ // failed sync with a body-prefetch storm into the same API:
1152
+ // a 429 on listLabels means Gmail is in cooldown, so firing
1153
+ // prefetch next would just re-trigger the same limit.
1154
+ return;
1155
+ }
1156
+ if (getPrefetch()) {
1157
+ this.prefetchBodies(accountId).catch(e => console.error(` [prefetch] ${accountId}: ${e.message}`));
1158
+ }
1159
+ };
1160
+ const syncPromises = [...this.configs.keys()].map(syncAndPrefetch);
1161
+ await Promise.allSettled(syncPromises);
1162
+ }
1163
+ /** Sync a single account — manages its own connection lifecycle */
1164
+ async syncAccount(accountId, priorityOrder) {
1165
+ // Gmail: use REST API instead of IMAP
1166
+ if (this.isGmailAccount(accountId)) {
1167
+ return this.syncAccountViaApi(accountId, priorityOrder);
1168
+ }
1169
+ try {
1170
+ // Step 1: Get folder list (fast — <1s typically)
1171
+ let client = await this.getOpsClient(accountId);
1172
+ const t0 = Date.now();
1173
+ const folders = await this.syncFolders(accountId, client);
1174
+ console.log(` [timing] ${accountId}: folder list ${Date.now() - t0}ms (${folders.length} folders)`);
1175
+ // Step 2: Sync INBOX first — keep retrying on failure (most important folder)
1176
+ const inbox = folders.find(f => f.specialUse === "inbox");
1177
+ if (inbox) {
1178
+ console.log(` [sync] ${accountId}: starting INBOX sync (folder ${inbox.id})`);
1179
+ const maxAttempts = 5;
1180
+ let inboxDone = false;
1181
+ for (let attempt = 1; attempt <= maxAttempts && !inboxDone; attempt++) {
1182
+ try {
1183
+ client = await this.getOpsClient(accountId);
1184
+ if (attempt > 1)
1185
+ console.log(` [sync] ${accountId}: INBOX retry #${attempt}`);
1186
+ await this.syncFolder(accountId, inbox.id, client);
1187
+ console.log(` [sync] ${accountId}: INBOX sync complete`);
1188
+ inboxDone = true;
1189
+ // Kick off prefetch as soon as INBOX is fresh, not
1190
+ // after all 105 folders finish — bobma's full sync
1191
+ // can take 30+ minutes on a wide folder tree, and
1192
+ // INBOX is the only folder the user is staring at.
1193
+ // Uses the body client (separate connection from
1194
+ // ops), so it runs concurrently with the rest of the
1195
+ // folder sync without contending for the same socket.
1196
+ if (getPrefetch()) {
1197
+ this.prefetchBodies(accountId).catch(e => console.error(` [prefetch] ${accountId}: ${e.message}`));
1198
+ }
1199
+ }
1200
+ catch (e) {
1201
+ console.error(` Inbox sync error for ${accountId} (attempt ${attempt}/${maxAttempts}): ${e.message}`);
1202
+ await this.reconnectOps(accountId);
1203
+ if (attempt < maxAttempts) {
1204
+ const delay = Math.min(attempt * 5000, 15000);
1205
+ console.log(` [sync] ${accountId}: waiting ${delay / 1000}s before INBOX retry`);
1206
+ await new Promise(r => setTimeout(r, delay));
1207
+ }
1208
+ }
1209
+ }
1210
+ if (!inboxDone) {
1211
+ console.error(` [sync] ${accountId}: INBOX failed after ${maxAttempts} attempts — will retry next sync cycle`);
1212
+ this.emit("syncError", accountId, `INBOX sync failed after ${maxAttempts} attempts`);
1213
+ // Even when sync failed, try to prefetch bodies for messages
1214
+ // already in the local DB. Prefetch uses a separate body
1215
+ // client (not the ops client that just timed out), so a
1216
+ // sync timeout on SELECT/SEARCH doesn't necessarily mean
1217
+ // body fetches will also fail. Without this, a server
1218
+ // having a slow patch would leave every message with a
1219
+ // white "not-downloaded" dot indefinitely until sync
1220
+ // recovers — even though prior syncs already populated
1221
+ // headers that prefetch can flesh out independently.
1222
+ if (getPrefetch()) {
1223
+ this.prefetchBodies(accountId).catch(e => console.error(` [prefetch] ${accountId}: ${e.message}`));
1224
+ }
1225
+ }
1226
+ }
1227
+ else {
1228
+ console.log(` [sync] ${accountId}: no INBOX folder found`);
1229
+ }
1230
+ // Step 3: Sync remaining folders.
1231
+ //
1232
+ // Parallel pool (concurrency 2) with a per-folder wall-clock cap.
1233
+ // Previous serial loop meant one slow Dovecot UID FETCH could park
1234
+ // every other folder behind it for minutes — user observed "mailx
1235
+ // says synced but 90 folders are empty" because the loop never
1236
+ // progressed past the stalled FETCH before the next sync tick.
1237
+ //
1238
+ // Parallelism uses independent IMAP sockets from the ops-client
1239
+ // pool, so one stalled socket doesn't block the others. The 60s
1240
+ // timeout abandons a stalled command instead of waiting out
1241
+ // Dovecot's 300s server-side inactivity timer; the next sync tick
1242
+ // retries on a fresh socket.
1243
+ const remaining = folders.filter(f => f.specialUse !== "inbox");
1244
+ remaining.sort((a, b) => {
1245
+ const pa = priorityOrder.indexOf(a.specialUse || "") >= 0 ? priorityOrder.indexOf(a.specialUse || "") : 5;
1246
+ const pb = priorityOrder.indexOf(b.specialUse || "") >= 0 ? priorityOrder.indexOf(b.specialUse || "") : 5;
1247
+ return pa - pb;
1248
+ });
1249
+ const CONCURRENCY = 2;
1250
+ const PER_FOLDER_TIMEOUT_MS = 60_000;
1251
+ const total = remaining.length;
1252
+ let done = 0;
1253
+ let idx = 0;
1254
+ const syncOne = async (folder) => {
1255
+ const isTrashChild = folder.path.includes("/") && folder.path.toLowerCase().startsWith("trash");
1256
+ const highestUid = this.db.getHighestUid(accountId, folder.id);
1257
+ if (isTrashChild && highestUid === 0)
1258
+ return;
1259
+ try {
1260
+ const fresh = await this.getOpsClient(accountId);
1261
+ await Promise.race([
1262
+ this.syncFolder(accountId, folder.id, fresh),
1263
+ new Promise((_, reject) => setTimeout(() => reject(new Error(`per-folder timeout (${PER_FOLDER_TIMEOUT_MS / 1000}s): ${folder.path}`)), PER_FOLDER_TIMEOUT_MS)),
1264
+ ]);
1265
+ }
1266
+ catch (e) {
1267
+ if (e.responseText?.includes("doesn't exist")) {
1268
+ this.db.deleteFolder(folder.id);
1269
+ }
1270
+ else {
1271
+ console.error(` Skipping ${folder.path}: ${e.message}`);
1272
+ this.recordError(accountId, e.message || String(e));
1273
+ // A timeout or stale-socket failure — drop the ops
1274
+ // client so the next iteration reconnects rather than
1275
+ // inheriting the doomed socket.
1276
+ await this.reconnectOps(accountId).catch(() => { });
1277
+ }
1278
+ }
1279
+ };
1280
+ const worker = async () => {
1281
+ while (true) {
1282
+ const myIdx = idx++;
1283
+ if (myIdx >= remaining.length)
1284
+ return;
1285
+ const folder = remaining[myIdx];
1286
+ this.emit("syncProgress", accountId, `folders:${folder.path}`, Math.round((done / Math.max(total, 1)) * 100));
1287
+ await syncOne(folder);
1288
+ done++;
1289
+ this.emit("syncProgress", accountId, `folders-done`, Math.round((done / Math.max(total, 1)) * 100));
1290
+ console.log(` [sync] ${accountId}: folder ${done}/${total} done (${folder.path})`);
1291
+ }
1292
+ };
1293
+ await Promise.all(Array.from({ length: Math.min(CONCURRENCY, remaining.length) }, () => worker()));
1294
+ this.accountErrorShown.delete(accountId);
1295
+ this.emit("syncComplete", accountId);
1296
+ }
1297
+ catch (e) {
1298
+ const errMsg = imapError(e);
1299
+ this.emit("syncError", accountId, errMsg);
1300
+ this.recordError(accountId, errMsg);
1301
+ console.error(`Sync error for ${accountId}: ${errMsg}`);
1302
+ this.handleSyncError(accountId, errMsg);
1303
+ }
1304
+ }
1305
+ /** Sync a Gmail account via REST API — no IMAP connections */
1306
+ async syncAccountViaApi(accountId, priorityOrder) {
1307
+ try {
1308
+ const api = this.getGmailProvider(accountId);
1309
+ const t0 = Date.now();
1310
+ // Step 1: Sync folder list via API
1311
+ console.log(` [api] ${accountId}: listing labels...`);
1312
+ const apiFolders = await api.listFolders();
1313
+ console.log(` [api] ${accountId}: ${apiFolders.length} labels in ${Date.now() - t0}ms`);
1314
+ // Store folders in DB (same as IMAP path)
1315
+ for (const f of apiFolders) {
1316
+ const specialUse = f.specialUse || "";
1317
+ this.db.upsertFolder(accountId, f.path, f.name, specialUse, f.delimiter);
1318
+ }
1319
+ this.emit("folderCountsChanged", accountId, {});
1320
+ const dbFolders = this.db.getFolders(accountId);
1321
+ // Step 2: Sync folders — INBOX first, then by priority
1322
+ const inbox = dbFolders.find(f => f.specialUse === "inbox");
1323
+ const remaining = dbFolders.filter(f => f.specialUse !== "inbox");
1324
+ remaining.sort((a, b) => {
1325
+ const pa = priorityOrder.indexOf(a.specialUse || "") >= 0 ? priorityOrder.indexOf(a.specialUse || "") : 5;
1326
+ const pb = priorityOrder.indexOf(b.specialUse || "") >= 0 ? priorityOrder.indexOf(b.specialUse || "") : 5;
1327
+ return pa - pb;
1328
+ });
1329
+ const foldersToSync = inbox ? [inbox, ...remaining] : remaining;
1330
+ for (const folder of foldersToSync) {
1331
+ try {
1332
+ await this.syncFolderViaApi(accountId, folder, api);
1333
+ }
1334
+ catch (e) {
1335
+ console.error(` [api] ${accountId}/${folder.path}: ${e.message}`);
1336
+ }
1337
+ }
1338
+ await api.close();
1339
+ this.accountErrorShown.delete(accountId);
1340
+ this.emit("syncComplete", accountId);
1341
+ }
1342
+ catch (e) {
1343
+ const errMsg = e.message || String(e);
1344
+ this.emit("syncError", accountId, errMsg);
1345
+ console.error(` [api] Sync error for ${accountId}: ${errMsg}`);
1346
+ this.handleSyncError(accountId, errMsg);
1347
+ // Propagate so the caller skips the prefetch that would otherwise
1348
+ // fire straight into the same 429/cooldown and make it worse.
1349
+ throw e;
1350
+ }
1351
+ }
1352
+ /** Sync a single folder via Gmail/Outlook API */
1353
+ async syncFolderViaApi(accountId, folder, api) {
1354
+ const highestUid = this.db.getHighestUid(accountId, folder.id);
1355
+ const historyDays = getHistoryDays(accountId);
1356
+ const effectiveDays = (historyDays === 0 && highestUid === 0) ? 30 : historyDays;
1357
+ const startDate = effectiveDays > 0 ? new Date(Date.now() - effectiveDays * 86400000) : new Date(0);
1358
+ const tomorrow = new Date(Date.now() + 86400000);
1359
+ this.emit("syncProgress", accountId, `sync:${folder.path}`, 0);
1360
+ console.log(` [api] ${accountId}/${folder.path}: syncing (highestUid=${highestUid})...`);
1361
+ let messages;
1362
+ if (highestUid > 0) {
1363
+ // Incremental: fetch messages since last known UID.
1364
+ // Gmail "UIDs" are hashed (not chronological), so fetchSince
1365
+ // returns messages in hash order — they can be from ANY date.
1366
+ // Pass the date window so the provider can page the whole range
1367
+ // (otherwise Gmail's default 200-id cap truncates high-volume
1368
+ // inboxes to ~10 days regardless of historyDays).
1369
+ const fetchOpts = { source: false };
1370
+ if (effectiveDays > 0)
1371
+ fetchOpts.since = startDate;
1372
+ messages = await api.fetchSince(folder.path, highestUid, fetchOpts);
1373
+ if (effectiveDays > 0) {
1374
+ const cutoff = startDate.getTime();
1375
+ const before = messages.length;
1376
+ messages = messages.filter(m => !m.date || m.date.getTime() >= cutoff);
1377
+ if (messages.length < before) {
1378
+ console.log(` [api] ${accountId}/${folder.path}: filtered ${before - messages.length} messages older than ${effectiveDays}d`);
1379
+ }
1380
+ }
1381
+ // Backfill: if the history window reaches further back than our
1382
+ // oldest local message, fetch the gap. Mirrors the IMAP path —
1383
+ // otherwise a user who started with historyDays=30 and later
1384
+ // sets it to 0 (or 365) never actually sees older mail. Cap
1385
+ // each sync cycle at 90 days so unlimited-history accounts
1386
+ // catch up incrementally instead of paging the whole mailbox.
1387
+ const oldestDate = this.db.getOldestDate(accountId, folder.id);
1388
+ if (oldestDate > 0 && startDate.getTime() < oldestDate) {
1389
+ try {
1390
+ const CHUNK_MS = 90 * 86400000;
1391
+ const chunkStart = new Date(Math.max(startDate.getTime(), oldestDate - CHUNK_MS));
1392
+ const existingUids = new Set(this.db.getUidsForFolder(accountId, folder.id));
1393
+ const backfill = await api.fetchByDate(folder.path, chunkStart, new Date(oldestDate), { source: false });
1394
+ const newBackfill = backfill.filter(m => !existingUids.has(m.uid));
1395
+ if (newBackfill.length > 0) {
1396
+ console.log(` [api] ${accountId}/${folder.path}: backfilling ${newBackfill.length} older messages (${chunkStart.toISOString().slice(0, 10)} → ${new Date(oldestDate).toISOString().slice(0, 10)})`);
1397
+ messages.push(...newBackfill);
1398
+ }
1399
+ }
1400
+ catch (e) {
1401
+ console.error(` [api] ${accountId}/${folder.path}: backfill failed: ${e.message}`);
1402
+ }
1403
+ }
1404
+ }
1405
+ else {
1406
+ // First sync: fetch by date range
1407
+ messages = await api.fetchByDate(folder.path, startDate, tomorrow, { source: false }, (chunk) => {
1408
+ // Stream chunks to DB for instant UI
1409
+ const stored = this.storeApiMessages(accountId, folder.id, chunk, highestUid);
1410
+ if (stored > 0) {
1411
+ this.db.recalcFolderCounts(folder.id);
1412
+ this.emit("folderCountsChanged", accountId, {});
1413
+ }
1414
+ });
1415
+ // First sync chunks already stored via onChunk — just update counts
1416
+ this.db.recalcFolderCounts(folder.id);
1417
+ this.emit("folderCountsChanged", accountId, {});
1418
+ this.emit("syncProgress", accountId, `sync:${folder.path}`, 100);
1419
+ if (messages.length > 0)
1420
+ console.log(` [api] ${accountId}/${folder.path}: ${messages.length} messages`);
1421
+ return;
1422
+ }
1423
+ if (messages.length > 0) {
1424
+ console.log(` [api] ${accountId}/${folder.path}: ${messages.length} new messages`);
1425
+ this.storeApiMessages(accountId, folder.id, messages, highestUid);
1426
+ }
1427
+ // Reconcile deletions — messages present locally but not on the server.
1428
+ // SAFETY: this used to silently wipe entire folders when getUids()
1429
+ // returned a partial list (e.g. paginated fetch hit a rate limit and
1430
+ // bailed). Multiple guards now:
1431
+ // 1. getUids() flags partial results via _truncated — refuse to delete
1432
+ // 2. If server list is empty but local isn't, assume a transient error
1433
+ // 3. If reconcile would delete more than RECONCILE_DELETE_THRESHOLD of
1434
+ // local messages, log and skip — safer to keep phantoms than to lose
1435
+ // real messages. User can fix with `mailx -rebuild` if needed.
1436
+ try {
1437
+ const serverUidsArr = await api.getUids(folder.path);
1438
+ const serverUids = new Set(serverUidsArr);
1439
+ const localUids = this.db.getUidsForFolder(accountId, folder.id);
1440
+ if (serverUidsArr._truncated) {
1441
+ console.log(` [api] ${accountId}/${folder.path}: reconcile skipped — server list truncated (${serverUidsArr.length} ids)`);
1442
+ }
1443
+ else if (serverUidsArr.length === 0 && localUids.length > 0) {
1444
+ console.log(` [api] ${accountId}/${folder.path}: reconcile skipped — server list empty but local has ${localUids.length}`);
1445
+ }
1446
+ else {
1447
+ const toDelete = localUids.filter(uid => !serverUids.has(uid));
1448
+ const RECONCILE_DELETE_THRESHOLD = 0.5; // refuse to delete >50% in one pass
1449
+ if (localUids.length > 0 && toDelete.length / localUids.length > RECONCILE_DELETE_THRESHOLD) {
1450
+ console.log(` [api] ${accountId}/${folder.path}: reconcile refused — would delete ${toDelete.length}/${localUids.length} (${Math.round(toDelete.length / localUids.length * 100)}%) — probably a sync bug, skipping`);
1451
+ }
1452
+ else {
1453
+ for (const uid of toDelete) {
1454
+ const env = this.db.getMessageByUid(accountId, uid);
1455
+ const tag = env ? `msgid=${env.messageId || "?"} subj="${(env.subject || "").slice(0, 60)}"` : "unknown";
1456
+ console.log(` [reconcile-delete] ${accountId}/${folder.path} uid=${uid} ${tag}`);
1457
+ this.unlinkBodyFile(accountId, uid, folder.id).catch(() => { });
1458
+ this.db.deleteMessage(accountId, uid);
1459
+ }
1460
+ if (toDelete.length > 0)
1461
+ console.log(` [api] ${accountId}/${folder.path}: ${toDelete.length} deleted`);
1462
+ }
1463
+ }
1464
+ }
1465
+ catch (e) {
1466
+ console.error(` [api] ${accountId}/${folder.path}: reconciliation error: ${e.message}`);
1467
+ }
1468
+ this.db.recalcFolderCounts(folder.id);
1469
+ this.emit("folderCountsChanged", accountId, {});
1470
+ this.emit("folderSynced", accountId, folder.id, Date.now());
1471
+ this.emit("syncProgress", accountId, `sync:${folder.path}`, 100);
1472
+ }
1473
+ /** Store API-fetched messages to DB */
1474
+ storeApiMessages(accountId, folderId, msgs, highestUid) {
1475
+ // highestUid kept for signature compatibility but no longer used to
1476
+ // filter — Gmail message IDs aren't monotonic, so `msg.uid <= highestUid`
1477
+ // would drop brand-new messages whose hash happens to be smaller than
1478
+ // the previous high. upsertMessage's primary-key dedup handles it.
1479
+ void highestUid;
1480
+ let stored = 0;
1481
+ let errors = 0;
1482
+ // Don't wrap the whole batch in one transaction: a single bad row
1483
+ // would roll back the entire batch. E.g. a message with a malformed
1484
+ // Date header gave `new Date(rawStr).getTime() === NaN`, SQLite
1485
+ // coerced that to NULL, the NOT NULL constraint failed, and the
1486
+ // whole Gmail sync lost 200 messages per tick. Now each row runs
1487
+ // standalone — bad rows are logged and skipped.
1488
+ for (const msg of msgs) {
1489
+ try {
1490
+ // Tombstone check — Gmail API sync was missing this so a
1491
+ // locally-trashed Gmail message would reappear on the next
1492
+ // listMessages tick if Gmail's eventual-consistency hadn't
1493
+ // promoted the trash yet. Symmetric with the IMAP paths.
1494
+ if (msg.messageId && this.db.hasTombstone(accountId, msg.messageId)) {
1495
+ continue;
1496
+ }
1497
+ const flags = [];
1498
+ if (msg.seen)
1499
+ flags.push("\\Seen");
1500
+ if (msg.flagged)
1501
+ flags.push("\\Flagged");
1502
+ if (msg.answered)
1503
+ flags.push("\\Answered");
1504
+ if (msg.draft)
1505
+ flags.push("\\Draft");
1506
+ // Sanitize date: reject NaN (from malformed RFC 822 Date headers)
1507
+ // and fall back to "now" so the message still lands in the DB.
1508
+ let dateMs = Date.now();
1509
+ if (msg.date instanceof Date) {
1510
+ const t = msg.date.getTime();
1511
+ if (Number.isFinite(t))
1512
+ dateMs = t;
1513
+ }
1514
+ this.db.upsertMessage({
1515
+ accountId, folderId, uid: msg.uid,
1516
+ messageId: msg.messageId || "",
1517
+ inReplyTo: msg.inReplyTo || "",
1518
+ references: msg.references || [],
1519
+ date: dateMs,
1520
+ subject: msg.subject || "",
1521
+ from: toEmailAddress(msg.from?.[0] || {}),
1522
+ to: toEmailAddresses(msg.to || []),
1523
+ cc: toEmailAddresses(msg.cc || []),
1524
+ flags, size: msg.size || 0, hasAttachments: false, preview: "", bodyPath: "",
1525
+ providerId: msg.providerId || "",
1526
+ });
1527
+ stored++;
1528
+ }
1529
+ catch (e) {
1530
+ errors++;
1531
+ if (errors <= 3) {
1532
+ console.error(` [api] upsert ${accountId}/${folderId}/${msg.uid} (${msg.messageId}): ${e.message}`);
1533
+ }
1534
+ }
1535
+ }
1536
+ if (errors > 0)
1537
+ console.error(` [api] storeApiMessages: ${errors} of ${msgs.length} rows failed (${stored} stored)`);
1538
+ return stored;
1539
+ }
1540
+ /** Kill and recreate the persistent ops connection */
1541
+ async reconnectOps(accountId) {
1542
+ const old = this.opsClients.get(accountId);
1543
+ this.opsClients.delete(accountId);
1544
+ if (old) {
1545
+ try {
1546
+ await (old._realLogout || old.logout)();
1547
+ }
1548
+ catch { /* */ }
1549
+ }
1550
+ console.log(` [conn] ${accountId}: reconnecting`);
1551
+ }
1552
+ /** Handle sync errors — classify and emit appropriate UI events.
1553
+ * The connection-cap branch was removed: with the unified ops queue +
1554
+ * per-host semaphore, mailx alone can't exceed the server cap. If the
1555
+ * cap *is* hit, that means another client (Thunderbird, phone, sibling
1556
+ * process) is holding slots — punishing mailx with a multi-minute
1557
+ * blackout doesn't help the user, the next sync tick will retry. */
1558
+ handleSyncError(accountId, errMsg) {
1559
+ const config = this.configs.get(accountId);
1560
+ const isOAuth = !!config?.tokenProvider;
1561
+ const isTransient = /timeout|ECONNREFUSED|ECONNRESET|ETIMEDOUT|ENETUNREACH|Too many/i.test(errMsg);
1562
+ const isAuth = /auth|login|credential|token|AUTHENTICATIONFAILED/i.test(errMsg);
1563
+ if (isTransient) {
1564
+ console.log(` [transient] ${accountId}: ${errMsg} — will retry next cycle`);
1565
+ }
1566
+ else if (isAuth && isOAuth) {
1567
+ const lastReauth = this.lastReauthAttempt.get(accountId) || 0;
1568
+ if (Date.now() - lastReauth > 300000 && !this.reauthenticating.has(accountId)) {
1569
+ this.lastReauthAttempt.set(accountId, Date.now());
1570
+ this.reauthenticate(accountId).catch(() => { });
1571
+ }
1572
+ if (!this.accountErrorShown.has(accountId)) {
1573
+ this.accountErrorShown.add(accountId);
1574
+ this.emit("accountError", accountId, errMsg, "Authentication expired — re-authenticate", true);
1575
+ }
1576
+ }
1577
+ else if (!this.accountErrorShown.has(accountId)) {
1578
+ this.accountErrorShown.add(accountId);
1579
+ this.emit("accountError", accountId, errMsg, errMsg, isOAuth);
1580
+ }
1581
+ }
1582
+ /** Fetch ONLY new messages above highestUid for one account's INBOX —
1583
+ * the IDLE callback's hot path. Skips gap detection, backfill, and the
1584
+ * server reconcile (each of which fetches a full UID list — multi-second
1585
+ * on a large mailbox). The 5-minute STATUS poll path still runs full
1586
+ * `syncFolder` so deletions and gaps eventually reconcile. */
1587
+ async syncInboxNewOnly(accountId) {
1588
+ if (this.isGmailAccount(accountId))
1589
+ return; // IDLE is IMAP-only
1590
+ const inbox = this.db.getFolders(accountId).find(f => f.specialUse === "inbox");
1591
+ if (!inbox)
1592
+ return;
1593
+ try {
1594
+ await this.withConnection(accountId, async (client) => {
1595
+ const highestUid = this.db.getHighestUid(accountId, inbox.id);
1596
+ if (highestUid === 0) {
1597
+ // First sync — fall through to full path so the date-windowed
1598
+ // backfill runs. `syncFolder` handles the no-highestUid case.
1599
+ await this.syncFolder(accountId, inbox.id, client);
1600
+ return;
1601
+ }
1602
+ const fetched = await client.fetchMessagesSinceUid(inbox.path, highestUid, { source: false });
1603
+ const fresh = fetched.filter((m) => m.uid > highestUid);
1604
+ if (fresh.length === 0)
1605
+ return;
1606
+ const stored = await this.storeMessages(accountId, inbox.id, inbox, fresh, highestUid);
1607
+ if (stored > 0) {
1608
+ this.db.recalcFolderCounts(inbox.id);
1609
+ const updated = this.db.getFolders(accountId).find(f => f.id === inbox.id);
1610
+ this.emit("folderCountsChanged", accountId, {
1611
+ [inbox.id]: { total: updated?.totalCount || 0, unread: updated?.unreadCount || 0 }
1612
+ });
1613
+ this.emit("folderSynced", accountId, inbox.id, Date.now());
1614
+ console.log(` [idle-fast] ${accountId}: stored ${stored} new message(s)`);
1615
+ }
1616
+ });
1617
+ }
1618
+ catch (e) {
1619
+ console.error(` [idle-fast] ${accountId}: ${e.message}`);
1620
+ }
1621
+ }
1622
+ /** Sync just INBOX for each account (fast check for new mail) */
1623
+ async syncInbox() {
1624
+ if (this.inboxSyncing)
1625
+ return;
1626
+ this.inboxSyncing = true;
1627
+ try {
1628
+ for (const [accountId] of this.configs) {
1629
+ try {
1630
+ const inbox = this.db.getFolders(accountId).find(f => f.specialUse === "inbox");
1631
+ if (!inbox)
1632
+ continue;
1633
+ // Gmail: use REST API, NOT IMAP. Mixing paths causes UID
1634
+ // mismatch — API uses hashed IDs, IMAP uses server-assigned
1635
+ // UIDs. The IMAP reconcile then deletes every API-synced
1636
+ // message because their UIDs don't appear in the IMAP list.
1637
+ if (this.isGmailAccount(accountId)) {
1638
+ const api = this.getGmailProvider(accountId);
1639
+ try {
1640
+ await this.syncFolderViaApi(accountId, inbox, api);
1641
+ }
1642
+ finally {
1643
+ try {
1644
+ await api.close();
1645
+ }
1646
+ catch { /* ignore */ }
1647
+ }
1648
+ }
1649
+ else {
1650
+ await this.withConnection(accountId, async (client) => {
1651
+ await this.syncFolder(accountId, inbox.id, client);
1652
+ });
1653
+ }
1654
+ }
1655
+ catch (e) {
1656
+ console.error(` [inbox] Sync error for ${accountId}: ${e.message}`);
1657
+ }
1658
+ }
1659
+ }
1660
+ finally {
1661
+ this.inboxSyncing = false;
1662
+ }
1663
+ }
1664
+ /** Quick inbox check — per-account lightweight probe.
1665
+ * If the probe value changed since last time, triggers an inbox sync.
1666
+ * The marker is only advanced after a successful sync so that a failed
1667
+ * sync doesn't eat the "new mail" signal and make us stop retrying. */
1668
+ lastInboxMarker = new Map();
1669
+ quickCheckRunning = new Set(); // per-account guard
1670
+ /** Shared quick-check skeleton: probe → compare → sync-if-changed → advance marker.
1671
+ * `probe` returns the current marker value; `sync` runs only when it differs
1672
+ * from the previously stored value. Marker is advanced only after sync resolves. */
1673
+ async quickCheck(accountId, probe, sync) {
1674
+ if (this.quickCheckRunning.has(accountId))
1675
+ return;
1676
+ if (this.reauthenticating.has(accountId))
1677
+ return;
1678
+ this.quickCheckRunning.add(accountId);
1679
+ try {
1680
+ const current = await probe();
1681
+ if (current === null || current === "")
1682
+ return;
1683
+ const prev = this.lastInboxMarker.get(accountId);
1684
+ if (prev === undefined || current !== prev) {
1685
+ await sync(current, prev);
1686
+ }
1687
+ // Only advance after sync succeeds — a thrown error skips this line
1688
+ // and the next tick will see the same delta and retry.
1689
+ this.lastInboxMarker.set(accountId, current);
1690
+ }
1691
+ catch {
1692
+ // Lightweight check — silently ignore errors
1693
+ }
1694
+ finally {
1695
+ this.quickCheckRunning.delete(accountId);
1696
+ }
1697
+ }
1698
+ /** Check a single account's inbox — uses its own connection, never blocked by sync */
1699
+ async quickInboxCheckAccount(accountId) {
1700
+ if (this.isGmailAccount(accountId))
1701
+ return this.quickGmailCheck(accountId);
1702
+ return this.quickImapCheck(accountId);
1703
+ }
1704
+ async quickImapCheck(accountId) {
1705
+ const inbox = this.db.getFolders(accountId).find(f => f.specialUse === "inbox");
1706
+ if (!inbox)
1707
+ return;
1708
+ let client = null;
1709
+ try {
1710
+ await this.quickCheck(accountId, async () => {
1711
+ client = await this.newClient(accountId, "quickCheck");
1712
+ return await client.getMessagesCount("INBOX");
1713
+ }, async (count, prev) => {
1714
+ if (prev !== undefined)
1715
+ console.log(` [check] ${accountId} INBOX: ${prev} → ${count}`);
1716
+ await this.syncFolder(accountId, inbox.id, client);
1717
+ });
1718
+ }
1719
+ finally {
1720
+ if (client) {
1721
+ try {
1722
+ await client.logout();
1723
+ }
1724
+ catch { /* */ }
1725
+ }
1726
+ }
1727
+ }
1728
+ async quickGmailCheck(accountId) {
1729
+ const config = this.configs.get(accountId);
1730
+ if (!config?.tokenProvider)
1731
+ return;
1732
+ const inbox = this.db.getFolders(accountId).find(f => f.specialUse === "inbox");
1733
+ if (!inbox)
1734
+ return;
1735
+ await this.quickCheck(accountId, async () => {
1736
+ const token = await config.tokenProvider();
1737
+ const res = await globalThis.fetch(`https://gmail.googleapis.com/gmail/v1/users/me/messages?q=in:inbox&maxResults=1`, { headers: { "Authorization": `Bearer ${token}` } });
1738
+ if (!res.ok)
1739
+ return null;
1740
+ const data = await res.json();
1741
+ return data.messages?.[0]?.id || null;
1742
+ }, async (_topId, prev) => {
1743
+ if (prev !== undefined)
1744
+ console.log(` [check] ${accountId} INBOX: new message detected`);
1745
+ const api = this.getGmailProvider(accountId);
1746
+ await this.syncFolderViaApi(accountId, inbox, api);
1747
+ this.db.recalcFolderCounts(inbox.id);
1748
+ this.emit("folderCountsChanged", accountId, {});
1749
+ await api.close();
1750
+ });
1751
+ }
1752
+ /** Check all accounts (used by legacy callers) */
1753
+ async quickInboxCheck() {
1754
+ for (const [accountId] of this.configs) {
1755
+ await this.quickInboxCheckAccount(accountId);
1756
+ }
1757
+ }
1758
+ /** Start periodic sync */
1759
+ startPeriodicSync(intervalMinutes) {
1760
+ this.stopPeriodicSync();
1761
+ // Per-account quick inbox check — adapts to server constraints.
1762
+ // Accounts with IDLE running get a long interval (5 min) because IDLE
1763
+ // already pushes instant notifications — the STATUS poll is just a
1764
+ // safety net. Non-IDLE accounts (rare) use a shorter interval.
1765
+ //
1766
+ // CRITICAL: the previous value (2500ms for everyone) was hammering
1767
+ // Dovecot with 24 logins per minute. That's what tripped the server
1768
+ // operator's fail2ban on mail1, and was still flooding the desktop
1769
+ // connection. Each STATUS poll creates a disposable connection
1770
+ // (TLS + auth + STATUS + close), not a lightweight keep-alive.
1771
+ for (const [accountId] of this.configs) {
1772
+ // Gmail uses API sync, not IMAP STATUS. IMAP accounts use IDLE
1773
+ // which gives instant push — the STATUS poll is just a fallback
1774
+ // in case IDLE silently dropped.
1775
+ const isGmail = this.isGmailAccount(accountId);
1776
+ // IMAP accounts: IDLE gives instant push; STATUS poll is just a
1777
+ // safety net for silent IDLE drops — keep it infrequent.
1778
+ // Gmail accounts: no IDLE (Gmail API doesn't expose it), so the
1779
+ // quick poll IS the primary path to new-mail latency. Drop to 30s
1780
+ // so Gmail mail appears in ~15s average. Gmail quota budget is
1781
+ // huge (250 units/sec per user, 1.2B/day) — 120 polls/hour × 5
1782
+ // units ≈ 600/hour, trivial. Dovecot accounts stay at 5min to
1783
+ // respect connection limits (each poll = fresh connection).
1784
+ const interval = isGmail ? 30000 : 300000; // Gmail: 30s; IMAP: 5min
1785
+ const timer = setInterval(() => {
1786
+ this.quickInboxCheckAccount(accountId).catch(() => { });
1787
+ }, interval);
1788
+ this.syncIntervals.set(`quick:${accountId}`, timer);
1789
+ console.log(` [periodic] ${accountId}: STATUS check every ${interval / 1000}s (${isGmail ? "API" : "IMAP+IDLE"})`);
1790
+ }
1791
+ // Sync actions (sends + flags/deletes/moves) every 30 seconds — skip during active sync
1792
+ const actionsInterval = setInterval(async () => {
1793
+ if (this.syncing)
1794
+ return;
1795
+ for (const [accountId] of this.configs) {
1796
+ this.processSendActions(accountId).catch(() => { });
1797
+ this.processSyncActions(accountId).catch(() => { });
1798
+ }
1799
+ }, 30000);
1800
+ this.syncIntervals.set("actions", actionsInterval);
1801
+ // Body prefetch as a first-class background task — independent of
1802
+ // sync success. Prefetch was previously only triggered from inside
1803
+ // sync, so any account with slow/failing IMAP had its "not downloaded"
1804
+ // dots stuck forever even though body fetches use a separate
1805
+ // connection that might succeed. Every 60s, for every account, fire
1806
+ // prefetchBodies() (cheap when body_path is already populated — just a
1807
+ // DB query that returns 0 rows; the prefetchingAccounts guard
1808
+ // short-circuits concurrent triggers).
1809
+ if (getPrefetch()) {
1810
+ const kickPrefetch = () => {
1811
+ for (const [accountId] of this.configs) {
1812
+ this.prefetchBodies(accountId).catch(e => console.error(` [prefetch] ${accountId}: ${e?.message || e}`));
1813
+ }
1814
+ };
1815
+ // Fire once now so the "not downloaded" dots start filling in
1816
+ // immediately on app start, don't make the user wait a minute.
1817
+ setTimeout(kickPrefetch, 2000);
1818
+ const prefetchInterval = setInterval(kickPrefetch, 60000);
1819
+ this.syncIntervals.set("prefetch", prefetchInterval);
1820
+ console.log(` [periodic] body prefetch every 60s (independent of sync)`);
1821
+ }
1822
+ // Tombstone prune: age out local-delete records older than 30 days.
1823
+ // Runs hourly — cheap (one indexed DELETE).
1824
+ const TOMBSTONE_RETENTION_DAYS = 30;
1825
+ const pruneTombstones = () => {
1826
+ const cutoff = Date.now() - TOMBSTONE_RETENTION_DAYS * 86400_000;
1827
+ const n = this.db.pruneTombstones(cutoff);
1828
+ if (n > 0)
1829
+ console.log(` [tombstones] pruned ${n} older than ${TOMBSTONE_RETENTION_DAYS} days`);
1830
+ };
1831
+ setTimeout(pruneTombstones, 30_000); // first run after startup settles
1832
+ this.syncIntervals.set("tombstone-prune", setInterval(pruneTombstones, 3600_000));
1833
+ // Full sync (all folders + IDLE restart) at configured interval
1834
+ const fullInterval = setInterval(async () => {
1835
+ console.log(` [periodic] Full sync at ${new Date().toLocaleTimeString()}`);
1836
+ await this.syncAll();
1837
+ await this.stopWatching();
1838
+ await this.startWatching();
1839
+ }, intervalMinutes * 60000);
1840
+ this.syncIntervals.set("all", fullInterval);
1841
+ }
1842
+ /** Stop periodic sync */
1843
+ stopPeriodicSync() {
1844
+ for (const [key, interval] of this.syncIntervals) {
1845
+ clearInterval(interval);
1846
+ }
1847
+ this.syncIntervals.clear();
1848
+ }
1849
+ /** Check if an account is OAuth (Gmail/Outlook — generous connection limits) */
1850
+ isOAuthAccount(accountId) {
1851
+ const config = this.configs.get(accountId);
1852
+ return !!config?.tokenProvider;
1853
+ }
1854
+ /** Start IMAP IDLE watchers for INBOX on each account */
1855
+ async startWatching() {
1856
+ for (const [accountId] of this.configs) {
1857
+ if (this.watchers.has(accountId))
1858
+ continue;
1859
+ try {
1860
+ // IDLE keeps its own dedicated socket — once the connection
1861
+ // is parked in IDLE, it's unusable for any other command, so
1862
+ // it can't share the ops queue. Counts against the per-host
1863
+ // semaphore (one slot for the IDLE socket).
1864
+ const watchClient = await this.createClient(accountId, "idle");
1865
+ const stop = await watchClient.watchMailbox("INBOX", (newCount) => {
1866
+ console.log(` [idle] ${accountId}: ${newCount} new message(s)`);
1867
+ // Fetch only the new UIDs — the heavyweight gap/reconcile
1868
+ // path runs on the 5-minute STATUS poll, so EXISTS lands
1869
+ // in the UI in roughly one round-trip.
1870
+ this.syncInboxNewOnly(accountId).catch(e => console.error(` [idle] sync error: ${e.message}`));
1871
+ });
1872
+ this.watchers.set(accountId, async () => {
1873
+ await stop();
1874
+ await watchClient.logout();
1875
+ });
1876
+ console.log(` [idle] Watching INBOX for ${accountId}`);
1877
+ }
1878
+ catch (e) {
1879
+ console.error(` [idle] Failed to watch ${accountId}: ${e.message}`);
1880
+ }
1881
+ }
1882
+ }
1883
+ /** Stop all IDLE watchers */
1884
+ async stopWatching() {
1885
+ for (const [id, stop] of this.watchers) {
1886
+ try {
1887
+ await stop();
1888
+ }
1889
+ catch { /* ignore */ }
1890
+ }
1891
+ this.watchers.clear();
1892
+ }
1893
+ /** Unlink the on-disk body file for a message by reading its `body_path`
1894
+ * from the DB. Safe to call either before or after `db.deleteMessage`
1895
+ * — read body_path first, store it, then unlink whenever. */
1896
+ async unlinkBodyFile(accountId, uid, folderId) {
1897
+ try {
1898
+ const row = this.db.getMessageByUid(accountId, uid, folderId);
1899
+ const p = row?.bodyPath;
1900
+ if (p)
1901
+ await this.bodyStore.unlinkByPath(p);
1902
+ }
1903
+ catch { /* row already gone / file already gone — both fine */ }
1904
+ }
1905
+ /** Fetch a single message body on demand, caching in the store.
1906
+ *
1907
+ * Cache lookup is folder-agnostic: when a UID exists in multiple folders
1908
+ * (Gmail labels, copy-instead-of-move) the prefetcher may have populated
1909
+ * body_path on only one row. Looking up by (account, uid) without the
1910
+ * folder filter finds the cached `.eml` regardless of which folder
1911
+ * context the UI passed.
1912
+ *
1913
+ * Server fetch goes through the unified ops queue on the fast lane —
1914
+ * the user clicked, they're waiting, this jumps ahead of any background
1915
+ * prefetch sitting in the slow lane. */
1916
+ async fetchMessageBody(accountId, folderId, uid) {
1917
+ const envelope = this.db.getMessageByUid(accountId, uid, folderId);
1918
+ let storedPath = envelope?.bodyPath || "";
1919
+ if (!storedPath)
1920
+ storedPath = this.db.getMessageBodyPath(accountId, uid) || "";
1921
+ if (storedPath && await this.bodyStore.hasByPath(storedPath)) {
1922
+ return this.bodyStore.readByPath(storedPath);
1923
+ }
1924
+ if (!this.configs.has(accountId))
1925
+ return null;
1926
+ const folder = this.db.getFolders(accountId).find(f => f.id === folderId);
1927
+ if (!folder)
1928
+ return null;
1929
+ // Gmail: REST API, no IMAP connection involved.
1930
+ if (this.isGmailAccount(accountId)) {
1931
+ return this.fetchMessageBodyViaApi(accountId, folderId, uid, folder.path);
1932
+ }
1933
+ // IMAP: fast lane on the ops queue. One try; if the socket is stale,
1934
+ // withConnection's discard-on-error logic drops the client so the
1935
+ // next attempt (caller-driven retry) gets a fresh one.
1936
+ try {
1937
+ const raw = await this.withConnection(accountId, async (client) => {
1938
+ const msg = await client.fetchMessageByUid(folder.path, uid, { source: true });
1939
+ if (!msg)
1940
+ throw makeNotFoundError(accountId, folderId, uid);
1941
+ if (!msg.source)
1942
+ return null;
1943
+ return Buffer.from(msg.source, "utf-8");
1944
+ });
1945
+ if (!raw)
1946
+ return null;
1947
+ const bodyPath = await this.bodyStore.putMessage(accountId, folderId, uid, raw);
1948
+ this.db.updateBodyPath(accountId, uid, bodyPath);
1949
+ this.emit("bodyCached", accountId, uid);
1950
+ return raw;
1951
+ }
1952
+ catch (e) {
1953
+ if (e?.isNotFound)
1954
+ throw e;
1955
+ console.error(` Body fetch error (${accountId}/${uid}): ${e?.message || e}`);
1956
+ return null;
1957
+ }
1958
+ }
1959
+ /** Fetch message body via Gmail/Outlook API.
1960
+ * Throws `MessageNotFoundError` when the server says the message is gone
1961
+ * (deleted from another device, for example). The caller uses that to
1962
+ * delete the stale row locally instead of showing a generic error. */
1963
+ async fetchMessageBodyViaApi(accountId, folderId, uid, folderPath) {
1964
+ try {
1965
+ const api = this.getGmailProvider(accountId);
1966
+ // Read provider_id from the local row so fetchOne can skip the
1967
+ // listMessageIds pagination (the dominant Gmail rate-limit cost).
1968
+ const env = this.db.getMessageByUid(accountId, uid, folderId);
1969
+ const providerId = env?.providerId;
1970
+ const msg = await api.fetchOne(folderPath, uid, { source: true, providerId });
1971
+ await api.close();
1972
+ if (!msg) {
1973
+ // fetchOne returned null — message doesn't exist on the server anymore
1974
+ throw makeNotFoundError(accountId, folderId, uid);
1975
+ }
1976
+ if (!msg.source) {
1977
+ // Gmail returned a message object but no raw bytes. Seen when:
1978
+ // (a) the message exists but is larger than the format=raw cap (~10MB),
1979
+ // (b) UID→Gmail-ID resolution picked a collision and the target
1980
+ // exists only as a stub, or (c) the listMessageIds top-1000
1981
+ // didn't include our UID and fetchOne returned null above —
1982
+ // wait, that would hit the !msg branch. So (a)/(b) remain.
1983
+ // Log enough to distinguish; surface the reason up via a non-null
1984
+ // return so the UI stops showing a generic "fetch returned nothing".
1985
+ console.error(` [api] Body fetch empty source (${accountId}/${uid}): Gmail returned no raw body — likely too-large-for-format-raw or UID hash collision`);
1986
+ return null;
1987
+ }
1988
+ const raw = Buffer.from(msg.source, "utf-8");
1989
+ const bodyPath = await this.bodyStore.putMessage(accountId, folderId, uid, raw);
1990
+ this.db.updateBodyPath(accountId, uid, bodyPath);
1991
+ this.emit("bodyCached", accountId, uid);
1992
+ return raw;
1993
+ }
1994
+ catch (e) {
1995
+ // Gmail API 404 → the message was deleted on the server
1996
+ if (e?.isNotFound || /Gmail API 404|404|not found/i.test(e?.message || "")) {
1997
+ throw makeNotFoundError(accountId, folderId, uid);
1998
+ }
1999
+ console.error(` [api] Body fetch error (${accountId}/${uid}): ${e.message}`);
2000
+ return null;
2001
+ }
2002
+ }
2003
+ /** Background body prefetch — download bodies for messages that don't have them.
2004
+ * Server-side deletions (isNotFound) aren't errors here: we delete the
2005
+ * stale row locally and keep going. Only unrelated errors (network,
2006
+ * auth, rate limits) count against the error budget, and the budget is
2007
+ * generous so a few transient failures don't kill the whole run. */
2008
+ /** Guard against concurrent prefetchBodies for the same account — mirror of
2009
+ * `sendingAccounts`. Without this, every periodic-sync tick spawns a new
2010
+ * prefetch session alongside any still in flight, blowing through Gmail's
2011
+ * per-minute quota and racing on disk writes. One prefetch per account. */
2012
+ prefetchingAccounts = new Set();
2013
+ /** Per-folder error cooldowns — `accountId:folderPath` → [timestamps].
2014
+ * Used to skip folders that repeatedly time out (Dovecot on slow shared
2015
+ * hosting SELECTs big `_Spam` / archive folders at 300s+ latency; the
2016
+ * prefetch error budget was burning out on a handful of bad folders
2017
+ * before the INBOX could finish). A folder with 2+ errors in the last
2018
+ * 15 minutes is skipped until the cooldown passes. User-reported via
2019
+ * log analysis 2026-04-23: bobma prefetch timing out on Added2.organizations,
2020
+ * Added2.technews, _Spam, "Prefirst.Jerry's Retreat", Added2.zines. */
2021
+ folderErrorCooldown = new Map();
2022
+ FOLDER_ERROR_WINDOW_MS = 15 * 60_000;
2023
+ FOLDER_ERROR_THRESHOLD = 2;
2024
+ shouldSkipFolder(accountId, folderPath) {
2025
+ const key = `${accountId}:${folderPath}`;
2026
+ const now = Date.now();
2027
+ const errors = (this.folderErrorCooldown.get(key) || [])
2028
+ .filter(t => now - t < this.FOLDER_ERROR_WINDOW_MS);
2029
+ this.folderErrorCooldown.set(key, errors);
2030
+ return errors.length >= this.FOLDER_ERROR_THRESHOLD;
2031
+ }
2032
+ recordFolderError(accountId, folderPath) {
2033
+ const key = `${accountId}:${folderPath}`;
2034
+ const arr = this.folderErrorCooldown.get(key) || [];
2035
+ arr.push(Date.now());
2036
+ this.folderErrorCooldown.set(key, arr);
2037
+ }
2038
+ clearFolderErrors(accountId, folderPath) {
2039
+ this.folderErrorCooldown.delete(`${accountId}:${folderPath}`);
2040
+ }
2041
+ async prefetchBodies(accountId) {
2042
+ if (this.prefetchingAccounts.has(accountId))
2043
+ return;
2044
+ this.prefetchingAccounts.add(accountId);
2045
+ try {
2046
+ await this._prefetchBodies(accountId);
2047
+ }
2048
+ finally {
2049
+ this.prefetchingAccounts.delete(accountId);
2050
+ }
2051
+ }
2052
+ async _prefetchBodies(accountId) {
2053
+ const counters = { totalFetched: 0, deleted: 0, errors: 0 };
2054
+ const ERROR_BUDGET = 20;
2055
+ const RATE_LIMIT_PAUSE_MS = 30000;
2056
+ const BATCH_SIZE = 100;
2057
+ const isGmail = this.isGmailAccount(accountId);
2058
+ // Gmail still uses per-message fetch (HTTP /batch is a separate TODO in this file's
2059
+ // governing unit). IMAP uses the batched `fetchBodiesBatch` path via iflow-direct —
2060
+ // one SELECT + one UID FETCH per folder per tick instead of N round trips.
2061
+ let announced = false;
2062
+ while (true) {
2063
+ const missing = this.db.getMessagesWithoutBody(accountId, BATCH_SIZE);
2064
+ if (missing.length === 0)
2065
+ break;
2066
+ if (!announced) {
2067
+ console.log(` [prefetch] ${accountId}: ${missing.length}+ bodies to fetch`);
2068
+ announced = true;
2069
+ }
2070
+ let madeProgress = false;
2071
+ if (isGmail) {
2072
+ // Gmail batch path: group by label (what mailx calls "folder"),
2073
+ // list once per label, bounded-concurrency fetch. Far fewer
2074
+ // HTTP round trips than the old one-listMessageIds-per-body path.
2075
+ // Note on the model: Gmail has labels, not folders. A message in
2076
+ // multiple labels gets fetched twice under current grouping. A
2077
+ // deeper label-native redesign is tracked as a separate TODO.
2078
+ const byFolder = new Map();
2079
+ for (const m of missing) {
2080
+ let arr = byFolder.get(m.folderId);
2081
+ if (!arr) {
2082
+ arr = [];
2083
+ byFolder.set(m.folderId, arr);
2084
+ }
2085
+ arr.push(m.uid);
2086
+ }
2087
+ const folders = this.db.getFolders(accountId);
2088
+ const api = this.getGmailProvider(accountId);
2089
+ try {
2090
+ for (const [folderId, uidsInFolder] of byFolder) {
2091
+ const folder = folders.find(f => f.id === folderId);
2092
+ if (!folder)
2093
+ continue;
2094
+ const received = new Set();
2095
+ const pending = [];
2096
+ let batchSucceeded = false;
2097
+ try {
2098
+ await api.fetchBodiesBatch(folder.path, uidsInFolder, (uid, source) => {
2099
+ received.add(uid);
2100
+ pending.push((async () => {
2101
+ try {
2102
+ const raw = Buffer.from(source, "utf-8");
2103
+ const bodyPath = await this.bodyStore.putMessage(accountId, folderId, uid, raw);
2104
+ this.db.updateBodyPath(accountId, uid, bodyPath);
2105
+ this.emit("bodyCached", accountId, uid);
2106
+ counters.totalFetched++;
2107
+ madeProgress = true;
2108
+ }
2109
+ catch (e) {
2110
+ console.error(` [prefetch] ${accountId}/${uid}: store write failed: ${e.message}`);
2111
+ }
2112
+ })());
2113
+ });
2114
+ batchSucceeded = true;
2115
+ }
2116
+ catch (e) {
2117
+ const isRate = /429|rate|too many/i.test(String(e?.message || ""));
2118
+ if (isRate) {
2119
+ console.log(` [prefetch] ${accountId}: rate-limited — pausing ${RATE_LIMIT_PAUSE_MS / 1000}s`);
2120
+ await new Promise(r => setTimeout(r, RATE_LIMIT_PAUSE_MS));
2121
+ }
2122
+ else {
2123
+ console.error(` [prefetch] ${accountId} folder ${folder.path}: Gmail batch fetch failed: ${e.message}`);
2124
+ counters.errors++;
2125
+ }
2126
+ }
2127
+ await Promise.all(pending);
2128
+ // CRITICAL: only prune as "server-deleted" when the batch
2129
+ // actually completed. If the batch threw (403, 429, network
2130
+ // error, etc.) NOTHING was received, and treating every
2131
+ // requested UID as deleted silently wipes 100 messages per
2132
+ // batch. That's a data-loss bug. Earlier version did this
2133
+ // and pruned 296 messages on a 403 auth error.
2134
+ if (batchSucceeded) {
2135
+ for (const uid of uidsInFolder) {
2136
+ if (received.has(uid))
2137
+ continue;
2138
+ try {
2139
+ this.unlinkBodyFile(accountId, uid, folderId).catch(() => { });
2140
+ this.db.deleteMessage(accountId, uid);
2141
+ counters.deleted++;
2142
+ madeProgress = true;
2143
+ }
2144
+ catch { /* ignore */ }
2145
+ }
2146
+ }
2147
+ if (counters.errors >= ERROR_BUDGET)
2148
+ break;
2149
+ }
2150
+ }
2151
+ finally {
2152
+ try {
2153
+ await api.close();
2154
+ }
2155
+ catch { /* ignore */ }
2156
+ }
2157
+ if (counters.errors >= ERROR_BUDGET) {
2158
+ console.error(` [prefetch] ${accountId}: stopping after ${counters.errors} errors (${counters.totalFetched} cached, ${counters.deleted} pruned)`);
2159
+ return;
2160
+ }
2161
+ }
2162
+ else {
2163
+ // IMAP batch path: one UID FETCH per folder, each on its own
2164
+ // turn through the slow lane. Yielding between folders is
2165
+ // crucial — a click-to-view body should jump ahead of the
2166
+ // next folder's batch via the fast lane, not wait for all
2167
+ // folders to finish.
2168
+ const byFolder = new Map();
2169
+ for (const m of missing) {
2170
+ let arr = byFolder.get(m.folderId);
2171
+ if (!arr) {
2172
+ arr = [];
2173
+ byFolder.set(m.folderId, arr);
2174
+ }
2175
+ arr.push(m.uid);
2176
+ }
2177
+ const folders = this.db.getFolders(accountId);
2178
+ // INBOX-first ordering so the folder the user actually looks at
2179
+ // gets its bodies even if a later folder eats the error budget.
2180
+ const orderedFolders = Array.from(byFolder.entries()).sort(([aid], [bid]) => {
2181
+ const af = folders.find(f => f.id === aid);
2182
+ const bf = folders.find(f => f.id === bid);
2183
+ const ai = af?.specialUse === "inbox" ? 0 : 1;
2184
+ const bi = bf?.specialUse === "inbox" ? 0 : 1;
2185
+ return ai - bi;
2186
+ });
2187
+ for (const [folderId, uids] of orderedFolders) {
2188
+ const folder = folders.find(f => f.id === folderId);
2189
+ if (!folder)
2190
+ continue;
2191
+ if (this.shouldSkipFolder(accountId, folder.path)) {
2192
+ console.log(` [prefetch] ${accountId}: skipping ${folder.path} (recent timeouts — cooling down)`);
2193
+ continue;
2194
+ }
2195
+ const received = new Set();
2196
+ let batchSucceeded = false;
2197
+ try {
2198
+ // Slow lane: prefetch is the textbook "this might take
2199
+ // a while" case — let interactive ops slip ahead.
2200
+ await this.withConnection(accountId, async (client) => {
2201
+ const pending = [];
2202
+ await client.fetchBodiesBatch(folder.path, uids, (uid, source) => {
2203
+ received.add(uid);
2204
+ pending.push((async () => {
2205
+ try {
2206
+ const raw = Buffer.from(source, "utf-8");
2207
+ const bodyPath = await this.bodyStore.putMessage(accountId, folderId, uid, raw);
2208
+ this.db.updateBodyPath(accountId, uid, bodyPath);
2209
+ this.emit("bodyCached", accountId, uid);
2210
+ counters.totalFetched++;
2211
+ madeProgress = true;
2212
+ }
2213
+ catch (e) {
2214
+ console.error(` [prefetch] ${accountId}/${uid}: store write failed: ${e.message}`);
2215
+ }
2216
+ })());
2217
+ });
2218
+ await Promise.all(pending);
2219
+ }, { slow: true });
2220
+ batchSucceeded = true;
2221
+ this.clearFolderErrors(accountId, folder.path);
2222
+ }
2223
+ catch (e) {
2224
+ const msg = String(e?.message || "");
2225
+ console.error(` [prefetch] ${accountId} folder ${folder.path}: batch fetch failed: ${msg}`);
2226
+ counters.errors++;
2227
+ this.recordFolderError(accountId, folder.path);
2228
+ if (counters.errors >= ERROR_BUDGET)
2229
+ break;
2230
+ }
2231
+ // CRITICAL: only prune when the batch actually completed.
2232
+ // A thrown batch means NOTHING was received; treating
2233
+ // absence as server-deletion lost 296 messages once.
2234
+ if (batchSucceeded)
2235
+ for (const uid of uids) {
2236
+ if (received.has(uid))
2237
+ continue;
2238
+ try {
2239
+ this.unlinkBodyFile(accountId, uid, folderId).catch(() => { });
2240
+ this.db.deleteMessage(accountId, uid);
2241
+ counters.deleted++;
2242
+ madeProgress = true;
2243
+ }
2244
+ catch { /* ignore */ }
2245
+ }
2246
+ }
2247
+ if (counters.errors >= ERROR_BUDGET) {
2248
+ console.error(` [prefetch] ${accountId}: stopping after ${counters.errors} errors (${counters.totalFetched} cached, ${counters.deleted} pruned)`);
2249
+ return;
2250
+ }
2251
+ }
2252
+ // Safety: zero progress this tick → bail rather than loop forever.
2253
+ if (!madeProgress)
2254
+ break;
2255
+ // Emit so the UI refreshes the open-circle → filled-teal indicator
2256
+ // without waiting for the next sync cycle.
2257
+ this.emit("folderCountsChanged", accountId, {});
2258
+ }
2259
+ if (counters.totalFetched > 0 || counters.deleted > 0) {
2260
+ console.log(` [prefetch] ${accountId}: ${counters.totalFetched} bodies cached, ${counters.deleted} stale rows pruned (done)`);
2261
+ this.emit("folderCountsChanged", accountId, {});
2262
+ }
2263
+ }
2264
+ /** Get the body store for direct access */
2265
+ getBodyStore() {
2266
+ return this.bodyStore;
2267
+ }
2268
+ /** Bulk trash messages — local-first, single IMAP connection for all */
2269
+ async trashMessages(accountId, messages) {
2270
+ if (messages.length === 0)
2271
+ return;
2272
+ const trash = this.findFolder(accountId, "trash");
2273
+ // Local first — remove all from DB immediately
2274
+ for (const msg of messages) {
2275
+ this.unlinkBodyFile(accountId, msg.uid, msg.folderId).catch(() => { });
2276
+ this.db.deleteMessage(accountId, msg.uid);
2277
+ }
2278
+ console.log(` Deleted ${messages.length} messages locally`);
2279
+ // Queue IMAP actions
2280
+ for (const msg of messages) {
2281
+ if (trash && trash.id !== msg.folderId) {
2282
+ this.db.queueSyncAction(accountId, "move", msg.uid, msg.folderId, { targetFolderId: trash.id });
2283
+ }
2284
+ else {
2285
+ this.db.queueSyncAction(accountId, "delete", msg.uid, msg.folderId);
2286
+ }
2287
+ }
2288
+ // Recalc folder counts so the tree badge updates immediately instead
2289
+ // of showing stale numbers until the next full sync.
2290
+ const sourceFolderIds = new Set(messages.map(m => m.folderId));
2291
+ for (const fid of sourceFolderIds)
2292
+ this.db.recalcFolderCounts(fid);
2293
+ if (trash)
2294
+ this.db.recalcFolderCounts(trash.id);
2295
+ this.emit("folderCountsChanged", accountId, {});
2296
+ // Process all queued actions in one IMAP session
2297
+ this.debounceSyncActions(accountId);
2298
+ }
2299
+ /** Bulk move messages — queues the IMAP action only. The service layer
2300
+ * (MailxService.moveMessages) owns the local DB mutation via
2301
+ * updateMessageFolder; this method used to ALSO deleteMessage here,
2302
+ * which wiped the row the service just updated — the message vanished
2303
+ * on the next reconcile and "spam folder empty" was the symptom. */
2304
+ async moveMessages(accountId, messages, targetFolderId) {
2305
+ if (messages.length === 0)
2306
+ return;
2307
+ for (const msg of messages) {
2308
+ this.db.queueSyncAction(accountId, "move", msg.uid, msg.folderId, { targetFolderId });
2309
+ }
2310
+ console.log(` [move] ${accountId}: queued IMAP MOVE for ${messages.length} message(s) → folder ${targetFolderId}`);
2311
+ this.debounceSyncActions(accountId);
2312
+ }
2313
+ /** Debounced sync actions — batches rapid local changes into one IMAP operation */
2314
+ syncActionTimers = new Map();
2315
+ debounceSyncActions(accountId) {
2316
+ const existing = this.syncActionTimers.get(accountId);
2317
+ if (existing)
2318
+ clearTimeout(existing);
2319
+ this.syncActionTimers.set(accountId, setTimeout(() => {
2320
+ this.syncActionTimers.delete(accountId);
2321
+ this.processSyncActions(accountId).catch(() => { });
2322
+ }, 1000));
2323
+ }
2324
+ /** Move a message to Trash (delete) — local-first, queues IMAP sync */
2325
+ async trashMessage(accountId, folderId, uid) {
2326
+ const trash = this.findFolder(accountId, "trash");
2327
+ // Local first — remove from DB immediately
2328
+ this.unlinkBodyFile(accountId, uid, folderId).catch(() => { });
2329
+ this.db.deleteMessage(accountId, uid);
2330
+ // Queue IMAP action + log the resolution so "I deleted a message and
2331
+ // now it's in neither trash nor deleted" is diagnosable from the log.
2332
+ if (trash && trash.id !== folderId) {
2333
+ const trashFolder = this.db.getFolders(accountId).find(f => f.id === trash.id);
2334
+ this.db.queueSyncAction(accountId, "move", uid, folderId, { targetFolderId: trash.id });
2335
+ console.log(` [trash] ${accountId} UID ${uid}: queued MOVE to "${trashFolder?.path || trash.path}" (id=${trash.id}, specialUse=trash)`);
2336
+ }
2337
+ else {
2338
+ this.db.queueSyncAction(accountId, "delete", uid, folderId);
2339
+ console.log(` [trash] ${accountId} UID ${uid}: queued EXPUNGE in folder ${folderId} (already in trash or no trash configured)`);
2340
+ }
2341
+ // Debounced sync — batches multiple deletes into one IMAP session
2342
+ this.debounceSyncActions(accountId);
2343
+ }
2344
+ /** Move a message between folders — queues IMAP sync only. Service
2345
+ * layer owns the local DB update (see MailxService.moveMessage). */
2346
+ async moveMessage(accountId, uid, fromFolderId, toFolderId) {
2347
+ this.db.queueSyncAction(accountId, "move", uid, fromFolderId, { targetFolderId: toFolderId });
2348
+ console.log(` [move] ${accountId}: queued IMAP MOVE UID ${uid} folder ${fromFolderId} → ${toFolderId}`);
2349
+ this.debounceSyncActions(accountId);
2350
+ }
2351
+ /** Move message across accounts using iflow's moveMessageToServer */
2352
+ async moveMessageCrossAccount(fromAccountId, uid, fromFolderId, toAccountId, toFolderId) {
2353
+ const fromFolders = this.db.getFolders(fromAccountId);
2354
+ const fromFolder = fromFolders.find(f => f.id === fromFolderId);
2355
+ if (!fromFolder)
2356
+ throw new Error(`Source folder ${fromFolderId} not found`);
2357
+ const toFolders = this.db.getFolders(toAccountId);
2358
+ const toFolder = toFolders.find(f => f.id === toFolderId);
2359
+ if (!toFolder)
2360
+ throw new Error(`Target folder ${toFolderId} not found`);
2361
+ // Two accounts, two ops connections. Cross-account move is rare
2362
+ // and requires both sockets to be live concurrently (we APPEND to
2363
+ // target while still authenticated to source), so this can't fold
2364
+ // into a single withConnection call.
2365
+ await this.withConnection(fromAccountId, async (sourceClient) => {
2366
+ await this.withConnection(toAccountId, async (targetClient) => {
2367
+ const msg = await sourceClient.fetchMessageByUid(fromFolder.path, uid, { source: true });
2368
+ if (!msg)
2369
+ throw new Error(`Message UID ${uid} not found in ${fromFolder.path}`);
2370
+ await sourceClient.moveMessageToServer(msg, fromFolder.path, targetClient, toFolder.path);
2371
+ this.db.deleteMessage(fromAccountId, uid);
2372
+ console.log(` Cross-account move: ${fromAccountId}/${fromFolder.path} UID ${uid} → ${toAccountId}/${toFolder.path}`);
2373
+ });
2374
+ });
2375
+ }
2376
+ /** Undelete — move from Trash back to original folder */
2377
+ async undeleteMessage(accountId, uid, originalFolderId) {
2378
+ const trash = this.findFolder(accountId, "trash");
2379
+ if (!trash)
2380
+ throw new Error("No Trash folder found");
2381
+ await this.moveMessage(accountId, uid, trash.id, originalFolderId);
2382
+ }
2383
+ /** Update flags — local-first, queues IMAP sync */
2384
+ async updateFlagsLocal(accountId, uid, folderId, flags) {
2385
+ this.db.updateMessageFlags(accountId, uid, flags);
2386
+ this.db.queueSyncAction(accountId, "flags", uid, folderId, { flags });
2387
+ // User-visible pink-dot pending state stays until the action drains.
2388
+ // The 30-second periodic tick was too slow — opening one message to
2389
+ // auto-mark-as-read left it pink for half a minute. Same 1-second
2390
+ // debounce as moves/deletes batches rapid flag churn without the
2391
+ // visual lag.
2392
+ this.debounceSyncActions(accountId);
2393
+ }
2394
+ /** Process pending sync actions for an account */
2395
+ async processSyncActions(accountId) {
2396
+ const actions = this.db.getPendingSyncActions(accountId);
2397
+ if (actions.length === 0)
2398
+ return;
2399
+ const startCount = actions.length;
2400
+ const folders = this.db.getFolders(accountId);
2401
+ // Gmail path: push flag/label changes through the REST provider so
2402
+ // they actually reach the server. Earlier this method always went
2403
+ // through withConnection → IMAP, which silently no-op'd for Gmail
2404
+ // accounts (REST-only, no IMAP connection) and left local-only stars
2405
+ // that vanished on the next full sync.
2406
+ if (this.isGmailAccount(accountId)) {
2407
+ const api = this.getGmailProvider(accountId);
2408
+ try {
2409
+ for (const action of actions) {
2410
+ const folder = folders.find(f => f.id === action.folderId);
2411
+ if (!folder) {
2412
+ this.db.completeSyncAction(action.id);
2413
+ continue;
2414
+ }
2415
+ try {
2416
+ if (action.action === "flags" && api.setFlags) {
2417
+ await api.setFlags(folder.path, action.uid, action.flags || []);
2418
+ console.log(` [api] ${accountId}: flags synced UID ${action.uid}`);
2419
+ }
2420
+ else if ((action.action === "delete" || action.action === "trash") && api.trashMessage) {
2421
+ await api.trashMessage(folder.path, action.uid);
2422
+ console.log(` [api] ${accountId}: trashed UID ${action.uid} from ${folder.path}`);
2423
+ }
2424
+ else if (action.action === "move" && api.moveMessage) {
2425
+ const target = folders.find(f => f.id === action.targetFolderId);
2426
+ if (!target) {
2427
+ // Unreachable target — drop the action rather than loop.
2428
+ console.error(` [api] ${accountId}: move target folder ${action.targetFolderId} missing — dropping UID ${action.uid}`);
2429
+ this.db.completeSyncAction(action.id);
2430
+ continue;
2431
+ }
2432
+ await api.moveMessage(folder.path, action.uid, target.path);
2433
+ console.log(` [api] ${accountId}: moved UID ${action.uid} ${folder.path} → ${target.path}`);
2434
+ }
2435
+ else {
2436
+ // Unsupported action on Gmail. After 5 retries, drop it
2437
+ // so stale rows don't mark messages pending-reconcile
2438
+ // forever. Previously "continue" here caused the pink
2439
+ // rows that shouldn't have been pink.
2440
+ if (action.attempts >= 5) {
2441
+ console.warn(` [api] ${accountId}: dropping stale action "${action.action}" UID ${action.uid} after ${action.attempts} attempts (unsupported on Gmail API path)`);
2442
+ this.db.completeSyncAction(action.id);
2443
+ }
2444
+ else {
2445
+ this.db.failSyncAction(action.id, `unsupported Gmail action: ${action.action}`);
2446
+ }
2447
+ continue;
2448
+ }
2449
+ this.db.completeSyncAction(action.id);
2450
+ }
2451
+ catch (e) {
2452
+ console.error(` [api] ${accountId}: flag sync failed UID ${action.uid}: ${e.message}`);
2453
+ this.db.failSyncAction(action.id, e.message);
2454
+ if (action.attempts >= 5)
2455
+ this.db.completeSyncAction(action.id);
2456
+ }
2457
+ }
2458
+ }
2459
+ finally {
2460
+ try {
2461
+ await api.close();
2462
+ }
2463
+ catch { /* */ }
2464
+ }
2465
+ // Nudge the UI so rows that were pending-reconcile re-query their
2466
+ // pending state (pink dot was sticky until this event fired).
2467
+ const remaining = this.db.getPendingSyncActions(accountId).length;
2468
+ if (remaining < startCount)
2469
+ this.emit("folderCountsChanged", accountId, {});
2470
+ return;
2471
+ }
2472
+ await this.withConnection(accountId, async (client) => {
2473
+ for (const action of actions) {
2474
+ const folder = folders.find(f => f.id === action.folderId);
2475
+ if (!folder) {
2476
+ this.db.completeSyncAction(action.id);
2477
+ continue;
2478
+ }
2479
+ try {
2480
+ switch (action.action) {
2481
+ case "delete":
2482
+ await client.deleteMessageByUid(folder.path, action.uid);
2483
+ console.log(` [sync] Deleted UID ${action.uid} from ${folder.path}`);
2484
+ break;
2485
+ case "move": {
2486
+ const target = folders.find(f => f.id === action.targetFolderId);
2487
+ if (!target) {
2488
+ // Target folder gone — treat as permanent failure so the
2489
+ // action doesn't loop forever. User must re-delete manually.
2490
+ console.error(` [sync] Move target folder ${action.targetFolderId} missing — dropping action UID ${action.uid}`);
2491
+ throw new Error(`move target folder ${action.targetFolderId} not found`);
2492
+ }
2493
+ const msg = await client.fetchMessageByUid(folder.path, action.uid, { source: false });
2494
+ if (!msg) {
2495
+ // Message no longer in source folder. Two real cases:
2496
+ // (a) another client already moved/deleted it — nothing to do,
2497
+ // just mark the action done.
2498
+ // (b) the server is lying (transient SELECT miss) — the retry
2499
+ // will pick it up. We can't tell these apart from one fetch,
2500
+ // so log loud and treat as (a) after the first failure; the
2501
+ // attempts counter handles (b) via the failSyncAction path.
2502
+ console.log(` [sync] Move UID ${action.uid} in ${folder.path}: message gone (attempt ${action.attempts + 1}); dropping action`);
2503
+ break;
2504
+ }
2505
+ await client.moveMessage(msg, folder.path, target.path);
2506
+ console.log(` [sync] Moved UID ${action.uid}: ${folder.path} → ${target.path}`);
2507
+ break;
2508
+ }
2509
+ case "flags":
2510
+ if (action.flags.length > 0) {
2511
+ await client.addFlags(folder.path, action.uid, action.flags.filter(f => !f.startsWith("-")));
2512
+ const toRemove = action.flags.filter(f => f.startsWith("-")).map(f => f.slice(1));
2513
+ if (toRemove.length > 0) {
2514
+ await client.removeFlags(folder.path, action.uid, toRemove);
2515
+ }
2516
+ console.log(` [sync] Updated flags UID ${action.uid}`);
2517
+ }
2518
+ break;
2519
+ case "append": {
2520
+ if (action.rawMessage) {
2521
+ await client.appendMessage(folder.path, action.rawMessage, action.flags);
2522
+ console.log(` [sync] Appended to ${folder.path}`);
2523
+ }
2524
+ break;
2525
+ }
2526
+ }
2527
+ this.db.completeSyncAction(action.id);
2528
+ }
2529
+ catch (e) {
2530
+ console.error(` [sync] Failed action ${action.action} UID ${action.uid}: ${e.message}`);
2531
+ this.db.failSyncAction(action.id, e.message);
2532
+ this.emit("syncActionFailed", accountId, action.action, action.uid, e.message);
2533
+ if (action.attempts >= 5) {
2534
+ console.error(` [sync] Giving up on action ${action.id} after 5 attempts`);
2535
+ this.db.completeSyncAction(action.id);
2536
+ this.emit("syncActionFailed", accountId, action.action, action.uid, `Gave up after 5 attempts: ${e.message}`);
2537
+ }
2538
+ }
2539
+ }
2540
+ });
2541
+ // IMAP path: same nudge as the Gmail branch above. Any action that
2542
+ // drained (successful or gave-up-after-5) decrements the pending
2543
+ // count, which flips the pink dot off on the next re-query.
2544
+ const remaining = this.db.getPendingSyncActions(accountId).length;
2545
+ if (remaining < startCount)
2546
+ this.emit("folderCountsChanged", accountId, {});
2547
+ }
2548
+ /** Find a folder by specialUse, case-insensitive */
2549
+ findFolder(accountId, specialUse) {
2550
+ const folders = this.db.getFolders(accountId);
2551
+ return folders.find(f => f.specialUse === specialUse ||
2552
+ f.path.toLowerCase() === specialUse.toLowerCase()) || null;
2553
+ }
2554
+ /** Optimistic local-first Sent insert: write a row into the local DB's
2555
+ * Sent folder the moment the user hits Send, so the list reflects it
2556
+ * immediately instead of waiting for SMTP + IMAP-APPEND + syncFolder
2557
+ * (five server round-trips against a Dovecot that caps at 20 conns).
2558
+ *
2559
+ * Uses a synthetic negative UID so it can't collide with a real APPENDUID
2560
+ * (which is always positive). When the real sync eventually picks the
2561
+ * message up in Sent with the server's UID, `db.upsertMessage` spots
2562
+ * the Message-ID match and rebinds the existing row's UID — no duplicate.
2563
+ * Negative UID also makes the row render pink (getMessages flags uid<0
2564
+ * as pending) so the user sees it's not-yet-reconciled.
2565
+ *
2566
+ * Best-effort — any failure path (no Sent folder yet, parse error, store
2567
+ * write error) is logged and swallowed; the send itself is unaffected. */
2568
+ async insertOptimisticSentRow(accountId, envelope, rawMessage) {
2569
+ try {
2570
+ const sent = this.findFolder(accountId, "sent");
2571
+ if (!sent) {
2572
+ console.log(` [sent-local] no Sent folder for ${accountId}; skipping optimistic row`);
2573
+ return;
2574
+ }
2575
+ // Synthetic UID — negative ms timestamp is monotonic + won't
2576
+ // collide with server UIDs. When the real APPENDUID returns via
2577
+ // sync, upsertMessage's Message-ID rebind swaps this for the
2578
+ // real positive value.
2579
+ const synthUid = -Date.now();
2580
+ const bodyPath = await this.bodyStore.putMessage(accountId, sent.id, synthUid, Buffer.from(rawMessage, "utf-8"));
2581
+ const parsed = await extractPreview(rawMessage);
2582
+ this.db.upsertMessage({
2583
+ accountId,
2584
+ folderId: sent.id,
2585
+ uid: synthUid,
2586
+ messageId: envelope.messageId,
2587
+ inReplyTo: envelope.inReplyTo,
2588
+ references: envelope.references,
2589
+ date: envelope.date,
2590
+ subject: envelope.subject,
2591
+ from: envelope.from,
2592
+ to: envelope.to,
2593
+ cc: envelope.cc,
2594
+ flags: ["\\Seen"],
2595
+ size: rawMessage.length,
2596
+ hasAttachments: parsed.hasAttachments,
2597
+ preview: parsed.preview,
2598
+ bodyPath,
2599
+ });
2600
+ // Folder-tree badge refresh + message-list reload if the user
2601
+ // is currently on Sent — same event the sync path emits.
2602
+ this.db.recalcFolderCounts(sent.id);
2603
+ this.emit("folderCountsChanged", { accountId, folderId: sent.id });
2604
+ console.log(` [sent-local] wrote optimistic row in Sent (uid=${synthUid}) for ${accountId}: ${envelope.subject}`);
2605
+ }
2606
+ catch (e) {
2607
+ // Non-fatal — send continues, Sent folder just won't show the
2608
+ // row until the real APPEND-then-sync cycle completes.
2609
+ console.error(` [sent-local] optimistic insert failed: ${e?.message || e}`);
2610
+ }
2611
+ }
2612
+ /** Copy sent message to the Sent folder via IMAP APPEND */
2613
+ async copyToSent(accountId, rawMessage) {
2614
+ const sent = this.findFolder(accountId, "sent");
2615
+ if (!sent) {
2616
+ console.error(` [sent] No Sent folder found for ${accountId}`);
2617
+ return;
2618
+ }
2619
+ await this.withConnection(accountId, async (client) => {
2620
+ await client.appendMessage(sent.path, rawMessage, ["\\Seen"]);
2621
+ console.log(` [sent] Copied to ${sent.path}`);
2622
+ });
2623
+ }
2624
+ /** Save a draft to the Drafts folder via IMAP APPEND.
2625
+ * Returns the UID of the saved draft (for replacing on next save). */
2626
+ async saveDraft(accountId, rawMessage, previousDraftUid, draftId) {
2627
+ const drafts = this.findFolder(accountId, "drafts");
2628
+ if (!drafts) {
2629
+ console.error(` [drafts] No Drafts folder found for ${accountId}`);
2630
+ return null;
2631
+ }
2632
+ return this.withConnection(accountId, async (client) => {
2633
+ // Delete previous draft — try UID first (fast path), and ALWAYS also try
2634
+ // searchByHeader(X-Mailx-Draft-ID) as a safety net. Running both catches
2635
+ // orphans from a crash-mid-save or a UID delete that failed silently.
2636
+ if (previousDraftUid) {
2637
+ try {
2638
+ await client.deleteMessageByUid(drafts.path, previousDraftUid);
2639
+ }
2640
+ catch (e) {
2641
+ console.error(` [drafts] Delete prev UID ${previousDraftUid} failed: ${e.message}`);
2642
+ }
2643
+ }
2644
+ if (draftId) {
2645
+ try {
2646
+ const uids = await client.searchByHeader(drafts.path, "X-Mailx-Draft-ID", draftId);
2647
+ for (const uid of uids) {
2648
+ try {
2649
+ await client.deleteMessageByUid(drafts.path, uid);
2650
+ }
2651
+ catch { /* next */ }
2652
+ }
2653
+ if (uids.length > 0)
2654
+ console.log(` [drafts] Deleted ${uids.length} stale draft(s) by ID ${draftId}`);
2655
+ }
2656
+ catch (e) {
2657
+ console.error(` [drafts] searchByHeader for ${draftId} failed: ${e.message}`);
2658
+ }
2659
+ }
2660
+ // Append new draft. If the server returns [TRYCREATE] (RFC 3501 §7.1),
2661
+ // the folder doesn't exist on the server even though mailx's DB has
2662
+ // it. Create it and retry once.
2663
+ let result;
2664
+ try {
2665
+ result = await client.appendMessage(drafts.path, rawMessage, ["\\Draft", "\\Seen"]);
2666
+ }
2667
+ catch (e) {
2668
+ const msg = String(e?.message || e);
2669
+ if (/TRYCREATE/i.test(msg)) {
2670
+ console.log(` [drafts] APPEND got TRYCREATE for "${drafts.path}" — creating folder and retrying`);
2671
+ try {
2672
+ await client.createmailbox(drafts.path);
2673
+ }
2674
+ catch (ce) {
2675
+ if (!/already exists/i.test(String(ce?.message || ""))) {
2676
+ console.error(` [drafts] Folder create failed for "${drafts.path}": ${ce.message}`);
2677
+ }
2678
+ }
2679
+ result = await client.appendMessage(drafts.path, rawMessage, ["\\Draft", "\\Seen"]);
2680
+ }
2681
+ else {
2682
+ throw e;
2683
+ }
2684
+ }
2685
+ const uid = typeof result === "number" ? result : result?.uid || null;
2686
+ return uid;
2687
+ });
2688
+ }
2689
+ /** Delete a draft (or all drafts with a stable X-Mailx-Draft-ID) after successful send.
2690
+ * Tries the specific UID first, then falls back to searchByHeader so orphaned copies
2691
+ * from earlier failed autosaves are cleaned up at the same time. */
2692
+ async deleteDraft(accountId, draftUid, draftId) {
2693
+ const drafts = this.findFolder(accountId, "drafts");
2694
+ if (!drafts)
2695
+ return;
2696
+ if (!draftUid && !draftId)
2697
+ return;
2698
+ await this.withConnection(accountId, async (client) => {
2699
+ if (draftUid) {
2700
+ try {
2701
+ await client.deleteMessageByUid(drafts.path, draftUid);
2702
+ console.log(` [drafts] Deleted draft UID ${draftUid}`);
2703
+ }
2704
+ catch (e) {
2705
+ console.error(` [drafts] Delete by UID ${draftUid} failed: ${e.message}`);
2706
+ }
2707
+ }
2708
+ if (draftId) {
2709
+ try {
2710
+ const uids = await client.searchByHeader(drafts.path, "X-Mailx-Draft-ID", draftId);
2711
+ for (const uid of uids) {
2712
+ try {
2713
+ await client.deleteMessageByUid(drafts.path, uid);
2714
+ }
2715
+ catch { /* next */ }
2716
+ }
2717
+ if (uids.length > 0)
2718
+ console.log(` [drafts] Deleted ${uids.length} draft(s) by ID ${draftId}`);
2719
+ }
2720
+ catch (e) {
2721
+ console.error(` [drafts] searchByHeader for ${draftId} failed: ${e.message}`);
2722
+ }
2723
+ }
2724
+ });
2725
+ }
2726
+ /** Queue outgoing message locally — never fails, worker handles IMAP+SMTP.
2727
+ * Single path: write `~/.mailx/outbox/<acct>/*.ltr` synchronously, then
2728
+ * kick processLocalQueue. The file IS the queue — durable across crashes,
2729
+ * visible in the filesystem, consumed by the existing outbox worker that
2730
+ * handles both IMAP-APPEND (non-Gmail) and direct SMTP (Gmail). The old
2731
+ * sync_actions "send" branch was removed because it duplicated the same
2732
+ * work and risked double-send when both paths fired on the same message. */
2733
+ queueOutgoingLocal(accountId, rawMessage) {
2734
+ // Loud logging so a "vanished message" report is diagnosable from the log alone.
2735
+ // ALWAYS leave a backup copy in sending/<acct>/attempted/ first — unconditionally,
2736
+ // before the outbox write. The outbox .ltr may be claimed/consumed by the worker
2737
+ // within milliseconds; this copy survives regardless of SMTP success, IMAP
2738
+ // append, worker crash, or any other downstream failure. User asked for this
2739
+ // as a fallback because "there isn't even the backup copy in sent".
2740
+ this.saveSendingCopy(accountId, rawMessage, "attempted");
2741
+ const outboxDir = path.join(getConfigDir(), "outbox", accountId);
2742
+ try {
2743
+ fs.mkdirSync(outboxDir, { recursive: true });
2744
+ }
2745
+ catch (e) {
2746
+ console.error(` [outbox] FAIL mkdirSync ${outboxDir}: ${e?.message || e}`);
2747
+ throw new Error(`Cannot create outbox dir ${outboxDir}: ${e?.message || e}`);
2748
+ }
2749
+ const now = new Date();
2750
+ const pad2 = (n) => String(n).padStart(2, "0");
2751
+ const filename = `${now.getFullYear()}${pad2(now.getMonth() + 1)}${pad2(now.getDate())}_${pad2(now.getHours())}${pad2(now.getMinutes())}${pad2(now.getSeconds())}-${String(Math.floor(Math.random() * 10000)).padStart(4, "0")}.ltr`;
2752
+ const filePath = path.join(outboxDir, filename);
2753
+ try {
2754
+ fs.writeFileSync(filePath, rawMessage);
2755
+ }
2756
+ catch (e) {
2757
+ console.error(` [outbox] FAIL writeFileSync ${filePath}: ${e?.message || e}`);
2758
+ throw new Error(`Cannot write outbox file ${filePath}: ${e?.message || e}`);
2759
+ }
2760
+ // Immediate readback verification — if this DOESN'T print, the user's
2761
+ // "neither in outbox nor file system" report has a real explanation.
2762
+ const written = fs.existsSync(filePath);
2763
+ const size = written ? fs.statSync(filePath).size : 0;
2764
+ console.log(` [outbox] WROTE ${filePath} (${size} bytes, exists=${written})`);
2765
+ this.emitOutboxStatus();
2766
+ // CRITICAL: defer to next tick. processLocalQueue runs ~30+ lines
2767
+ // of synchronous fs work BEFORE its first await — calling it inline
2768
+ // blocks the IPC ack on all that work.
2769
+ setImmediate(() => {
2770
+ this.processLocalQueue(accountId)
2771
+ .catch((e) => console.error(` [outbox] processLocalQueue error: ${e?.message || e}`))
2772
+ .finally(() => this.emitOutboxStatus());
2773
+ });
2774
+ }
2775
+ /** Scan the local outbox + sending/queued dirs and return counts + age.
2776
+ * Cheap — a handful of readdir + head-read per file. Called by both the
2777
+ * polling UI (status bar) and emitted as an event after queue mutations. */
2778
+ getOutboxStatus() {
2779
+ const configDir = getConfigDir();
2780
+ const perAccount = {};
2781
+ let total = 0;
2782
+ let retrying = 0;
2783
+ let claimed = 0;
2784
+ let oldestMs = 0;
2785
+ let maxAttempts = 0;
2786
+ const now = Date.now();
2787
+ const scan = (accountId, dir) => {
2788
+ if (!fs.existsSync(dir))
2789
+ return;
2790
+ for (const f of fs.readdirSync(dir)) {
2791
+ const isClaim = /\.sending-[^-]+-\d+$/.test(f);
2792
+ const isActive = isClaim || f.endsWith(".ltr") || f.endsWith(".eml");
2793
+ if (!isActive)
2794
+ continue;
2795
+ total++;
2796
+ const acctSlot = perAccount[accountId] ||= { total: 0, retrying: 0, claimed: 0 };
2797
+ acctSlot.total++;
2798
+ if (isClaim) {
2799
+ claimed++;
2800
+ acctSlot.claimed++;
2801
+ }
2802
+ const fp = path.join(dir, f);
2803
+ try {
2804
+ const st = fs.statSync(fp);
2805
+ const age = now - st.mtimeMs;
2806
+ if (age > oldestMs)
2807
+ oldestMs = age;
2808
+ // Only read header region to count retry attempts — tiny I/O.
2809
+ const fd = fs.openSync(fp, "r");
2810
+ try {
2811
+ const buf = Buffer.alloc(4096);
2812
+ const n = fs.readSync(fd, buf, 0, 4096, 0);
2813
+ const head = buf.slice(0, n).toString("utf-8");
2814
+ const info = parseRetryInfo(head);
2815
+ if (info.attemptCount > 0) {
2816
+ retrying++;
2817
+ acctSlot.retrying++;
2818
+ }
2819
+ if (info.attemptCount > maxAttempts)
2820
+ maxAttempts = info.attemptCount;
2821
+ }
2822
+ finally {
2823
+ fs.closeSync(fd);
2824
+ }
2825
+ }
2826
+ catch { /* ignore per-file errors */ }
2827
+ }
2828
+ };
2829
+ const outboxRoot = path.join(configDir, "outbox");
2830
+ const sendingRoot = path.join(configDir, "sending");
2831
+ try {
2832
+ if (fs.existsSync(outboxRoot)) {
2833
+ for (const acct of fs.readdirSync(outboxRoot))
2834
+ scan(acct, path.join(outboxRoot, acct));
2835
+ }
2836
+ if (fs.existsSync(sendingRoot)) {
2837
+ for (const acct of fs.readdirSync(sendingRoot)) {
2838
+ scan(acct, path.join(sendingRoot, acct, "queued"));
2839
+ }
2840
+ }
2841
+ }
2842
+ catch { /* */ }
2843
+ return {
2844
+ total, retrying, claimed,
2845
+ oldestAgeSec: Math.floor(oldestMs / 1000),
2846
+ maxAttempts,
2847
+ perAccount,
2848
+ };
2849
+ }
2850
+ /** Emit outboxStatus now. Call after any queue mutation. */
2851
+ emitOutboxStatus() {
2852
+ try {
2853
+ this.emit("outboxStatus", this.getOutboxStatus());
2854
+ }
2855
+ catch { /* */ }
2856
+ }
2857
+ /** Guard against concurrent processSendActions for the same account */
2858
+ sendingAccounts = new Set();
2859
+ /** Process local send actions — APPEND to Outbox, which the outbox worker then sends */
2860
+ async processSendActions(accountId) {
2861
+ if (this.sendingAccounts.has(accountId))
2862
+ return; // already processing
2863
+ this.sendingAccounts.add(accountId);
2864
+ try {
2865
+ await this._processSendActions(accountId);
2866
+ }
2867
+ finally {
2868
+ this.sendingAccounts.delete(accountId);
2869
+ }
2870
+ }
2871
+ async _processSendActions(accountId) {
2872
+ const actions = this.db.getPendingSyncActions(accountId)
2873
+ .filter(a => a.action === "send");
2874
+ if (actions.length === 0)
2875
+ return;
2876
+ for (const action of actions) {
2877
+ if (!action.rawMessage) {
2878
+ this.db.completeSyncAction(action.id);
2879
+ continue;
2880
+ }
2881
+ // Abandon after 10 failed attempts — don't retry forever
2882
+ if (action.attempts >= 10) {
2883
+ console.error(` [outbox] Abandoning send action ${action.id} after ${action.attempts} attempts: ${action.rawMessage?.substring(0, 100)}`);
2884
+ this.db.completeSyncAction(action.id);
2885
+ this.emit("accountError", accountId, `Send permanently failed after ${action.attempts} attempts`, "Message removed from queue", false);
2886
+ continue;
2887
+ }
2888
+ try {
2889
+ await this.queueOutgoing(accountId, action.rawMessage);
2890
+ this.db.completeSyncAction(action.id);
2891
+ }
2892
+ catch (e) {
2893
+ console.error(` [outbox] Local→IMAP failed (attempt ${action.attempts + 1}): ${e.message}`);
2894
+ this.db.failSyncAction(action.id, e.message);
2895
+ }
2896
+ }
2897
+ }
2898
+ // ── Outbox ──
2899
+ outboxInterval = null;
2900
+ hostname = os.hostname();
2901
+ /** Ensure Outbox folder exists, create if needed */
2902
+ async ensureOutbox(accountId) {
2903
+ let outbox = this.findFolder(accountId, "outbox");
2904
+ if (outbox)
2905
+ return outbox.path;
2906
+ // Look for existing folder named Outbox (case-insensitive)
2907
+ const folders = this.db.getFolders(accountId);
2908
+ const existing = folders.find(f => f.path.toLowerCase() === "outbox");
2909
+ if (existing)
2910
+ return existing.path;
2911
+ try {
2912
+ await this.withConnection(accountId, async (client) => {
2913
+ await client.createmailbox("Outbox");
2914
+ await this.syncFolders(accountId, client);
2915
+ });
2916
+ }
2917
+ catch (e) {
2918
+ // Might already exist — benign
2919
+ if (!e.message?.includes("already exists"))
2920
+ throw e;
2921
+ }
2922
+ outbox = this.findFolder(accountId, "outbox");
2923
+ return outbox?.path || "Outbox";
2924
+ }
2925
+ /** Save a copy of outgoing mail — label is a subdirectory (editing/queued/sent) */
2926
+ saveSendingCopy(accountId, rawMessage, label) {
2927
+ try {
2928
+ const dir = path.join(getConfigDir(), "sending", accountId, label);
2929
+ fs.mkdirSync(dir, { recursive: true });
2930
+ const now = new Date();
2931
+ const pad2 = (n) => String(n).padStart(2, "0");
2932
+ const ts = `${now.getFullYear()}${pad2(now.getMonth() + 1)}${pad2(now.getDate())}_${pad2(now.getHours())}${pad2(now.getMinutes())}${pad2(now.getSeconds())}`;
2933
+ fs.writeFileSync(path.join(dir, `${ts}.eml`), rawMessage);
2934
+ console.log(` [sending] ${label}/${ts}.eml`);
2935
+ }
2936
+ catch (e) {
2937
+ console.error(` [sending] Failed to save copy: ${e.message}`);
2938
+ }
2939
+ }
2940
+ /** Queue a message for sending. Tries IMAP Outbox, falls back to local file. */
2941
+ async queueOutgoing(accountId, rawMessage) {
2942
+ // IMPORTANT: do NOT save a "debug copy" to sending/<acct>/queued/ here.
2943
+ // processLocalQueue also scans sending/<acct>/queued/, so writing there
2944
+ // on every send caused the same message to be re-APPENDed to the IMAP
2945
+ // Outbox on the next outbox tick — resulting in a duplicate send.
2946
+ // The only two legitimate queue locations are:
2947
+ // - IMAP Outbox (primary, populated by APPEND below)
2948
+ // - ~/.mailx/outbox/<acct>/*.ltr (fallback when IMAP is unreachable)
2949
+ try {
2950
+ const outboxPath = await this.ensureOutbox(accountId);
2951
+ await this.withConnection(accountId, async (client) => {
2952
+ await client.appendMessage(outboxPath, rawMessage, ["\\Seen"]);
2953
+ console.log(` [outbox] Queued message in ${outboxPath}`);
2954
+ });
2955
+ const outboxFolder = this.findFolder(accountId, "outbox");
2956
+ if (outboxFolder) {
2957
+ this.syncFolder(accountId, outboxFolder.id).catch(() => { });
2958
+ }
2959
+ return;
2960
+ }
2961
+ catch (e) {
2962
+ console.error(` [outbox] IMAP queue failed: ${e.message} — saving locally`);
2963
+ }
2964
+ // Fallback: save to local file queue (processLocalQueue picks these up)
2965
+ const localQueue = path.join(getConfigDir(), "outbox", accountId);
2966
+ fs.mkdirSync(localQueue, { recursive: true });
2967
+ const now = new Date();
2968
+ const pad2 = (n) => String(n).padStart(2, "0");
2969
+ const filename = `${now.getFullYear()}${pad2(now.getMonth() + 1)}${pad2(now.getDate())}_${pad2(now.getHours())}${pad2(now.getMinutes())}${pad2(now.getSeconds())}-${String(Math.floor(Math.random() * 10000)).padStart(4, "0")}.ltr`;
2970
+ fs.writeFileSync(path.join(localQueue, filename), rawMessage);
2971
+ console.log(` [outbox] Saved locally: ${filename}`);
2972
+ }
2973
+ /** Process local file queue — scan both outbox/ (IMAP-unreachable fallback)
2974
+ * and sending/<acct>/queued/ (manual drop-in, crash recovery). The earlier
2975
+ * double-send bug was caused by queueOutgoing() WRITING a debug copy to
2976
+ * sending/queued/ on every send — that write is gone now, so scanning the
2977
+ * directory is safe again. Any legitimate files that land there (crash
2978
+ * recovery, manual drop) will get sent. */
2979
+ async processLocalQueue(accountId) {
2980
+ const outboxDir = path.join(getConfigDir(), "outbox", accountId);
2981
+ const queuedDir = path.join(getConfigDir(), "sending", accountId, "queued");
2982
+ // Recovery sweep: any *.sending-<host>-<pid> on THIS host whose PID is
2983
+ // dead (process crashed mid-send) gets unclaimed so the next tick can
2984
+ // retry. Foreign hosts are left alone — we have no way to know if their
2985
+ // process is alive. Cross-host stale recovery is the IMAP-folder path's
2986
+ // job (sweeper looks at server-side claim flags, not local files).
2987
+ for (const dir of [outboxDir, queuedDir]) {
2988
+ if (!fs.existsSync(dir))
2989
+ continue;
2990
+ for (const f of fs.readdirSync(dir)) {
2991
+ const m = f.match(/^(.+)\.sending-([^-]+)-(\d+)$/);
2992
+ if (!m)
2993
+ continue;
2994
+ const [, original, host, pidStr] = m;
2995
+ if (host !== this.hostname)
2996
+ continue;
2997
+ const pid = parseInt(pidStr);
2998
+ let alive = false;
2999
+ try {
3000
+ process.kill(pid, 0);
3001
+ alive = true;
3002
+ }
3003
+ catch { /* dead */ }
3004
+ if (alive)
3005
+ continue; // live claim — owner (sibling or self) still has it
3006
+ try {
3007
+ fs.renameSync(path.join(dir, f), path.join(dir, original));
3008
+ console.log(` [outbox] Recovered stale claim ${f} → ${original}`);
3009
+ }
3010
+ catch { /* ignore */ }
3011
+ }
3012
+ }
3013
+ const filesToSend = [];
3014
+ for (const dir of [outboxDir, queuedDir]) {
3015
+ if (!fs.existsSync(dir))
3016
+ continue;
3017
+ for (const file of fs.readdirSync(dir).filter(f => f.endsWith(".ltr") || f.endsWith(".eml"))) {
3018
+ filesToSend.push({ dir, file });
3019
+ }
3020
+ }
3021
+ if (filesToSend.length === 0)
3022
+ return;
3023
+ // Gmail/API accounts: send directly via SMTP from local queue (no IMAP outbox)
3024
+ const sentDir = path.join(getConfigDir(), "sending", accountId, "sent");
3025
+ fs.mkdirSync(sentDir, { recursive: true });
3026
+ if (this.isGmailAccount(accountId)) {
3027
+ const nowMs = Date.now();
3028
+ for (const { dir, file } of filesToSend) {
3029
+ const filePath = path.join(dir, file);
3030
+ // Atomic claim: rename to <file>.sending-<host>-<pid> so a sibling
3031
+ // process scanning the same dir can't grab the same .ltr. Filesystem
3032
+ // rename is atomic; loser sees ENOENT and skips. Without this, two
3033
+ // mailx instances on one machine (or two ticks within one process)
3034
+ // could both pass the Message-ID dedup check and both call SMTP.
3035
+ const claimSuffix = `.sending-${this.hostname}-${process.pid}`;
3036
+ const claimedPath = filePath + claimSuffix;
3037
+ try {
3038
+ fs.renameSync(filePath, claimedPath);
3039
+ }
3040
+ catch (e) {
3041
+ if (e.code === "ENOENT")
3042
+ continue; // another process won
3043
+ throw e;
3044
+ }
3045
+ let raw = fs.readFileSync(claimedPath, "utf-8");
3046
+ // Per-message rate limit: if a prior attempt set X-Mailx-Retry-After
3047
+ // in the future, skip this file for now. Minimizes the race where the
3048
+ // SMTP server actually accepted DATA but we lost the ack and would
3049
+ // otherwise retry immediately on the next 10s tick.
3050
+ const retryInfo = parseRetryInfo(raw);
3051
+ if (retryInfo.nextAttemptAt > nowMs) {
3052
+ // Release claim — let next tick reconsider
3053
+ try {
3054
+ fs.renameSync(claimedPath, filePath);
3055
+ }
3056
+ catch { /* ignore */ }
3057
+ continue;
3058
+ }
3059
+ // Record this attempt: strip internal X-Mailx-Retry-After, append a new
3060
+ // X-Mailx-Retry: N <ISO-timestamp> line to the headers. The updated file
3061
+ // is written back *before* the send so a crash mid-send doesn't lose state.
3062
+ const attempt = retryInfo.attemptCount + 1;
3063
+ raw = stripHeaderField(raw, "X-Mailx-Retry-After");
3064
+ raw = insertHeaderBeforeBody(raw, `X-Mailx-Retry: ${attempt} ${new Date().toISOString()}`);
3065
+ fs.writeFileSync(claimedPath, raw, "utf-8");
3066
+ try {
3067
+ await this.sendRawViaSMTP(accountId, raw);
3068
+ fs.renameSync(claimedPath, path.join(sentDir, file));
3069
+ console.log(` [outbox] Sent ${file} via SMTP → sent/ (attempt ${attempt})`);
3070
+ }
3071
+ catch (e) {
3072
+ // Persist a next-attempt timestamp and release the claim so the
3073
+ // file is visible to the scan loop again.
3074
+ const nextAt = new Date(nowMs + OUTBOX_RETRY_DELAY_MS).toISOString();
3075
+ const withDelay = insertHeaderBeforeBody(raw, `X-Mailx-Retry-After: ${nextAt}`);
3076
+ fs.writeFileSync(claimedPath, withDelay, "utf-8");
3077
+ try {
3078
+ fs.renameSync(claimedPath, filePath);
3079
+ }
3080
+ catch { /* file stays claimed; recovery sweeper will handle */ }
3081
+ console.error(` [outbox] Send failed for ${file} (attempt ${attempt}, retry after ${nextAt}): ${e.message}`);
3082
+ }
3083
+ }
3084
+ return;
3085
+ }
3086
+ // IMAP accounts: append to IMAP Outbox for multi-machine interlock.
3087
+ //
3088
+ // Atomic claim (same pattern as the Gmail path above): rename the file
3089
+ // to <file>.sending-<host>-<pid> BEFORE reading it, so two concurrent
3090
+ // mailx instances scanning the same dir can't both APPEND the same
3091
+ // message to IMAP Outbox and end up with a duplicate. Filesystem rename
3092
+ // is atomic; the loser sees ENOENT and skips. On APPEND success, move
3093
+ // the claimed file to sending/sent/; on APPEND failure, release the
3094
+ // claim so the recovery sweeper picks it up next tick.
3095
+ try {
3096
+ const outboxPath = await this.ensureOutbox(accountId);
3097
+ const client = await this.createClientWithLimit(accountId);
3098
+ try {
3099
+ for (const { dir, file } of filesToSend) {
3100
+ const filePath = path.join(dir, file);
3101
+ const claimSuffix = `.sending-${this.hostname}-${process.pid}`;
3102
+ const claimedPath = filePath + claimSuffix;
3103
+ try {
3104
+ fs.renameSync(filePath, claimedPath);
3105
+ }
3106
+ catch (e) {
3107
+ if (e.code === "ENOENT")
3108
+ continue; // sibling claimed first
3109
+ throw e;
3110
+ }
3111
+ try {
3112
+ const raw = fs.readFileSync(claimedPath, "utf-8");
3113
+ await client.appendMessage(outboxPath, raw, ["\\Seen"]);
3114
+ fs.renameSync(claimedPath, path.join(sentDir, file));
3115
+ console.log(` [outbox] Moved ${file} to IMAP Outbox → sent/`);
3116
+ }
3117
+ catch (e) {
3118
+ // APPEND failed (connection dropped mid-send, server
3119
+ // busy, etc.) — release the claim so next tick can
3120
+ // retry. Don't swallow: rethrow after release so the
3121
+ // outer catch ("IMAP still unreachable") bails out of
3122
+ // the remaining files too — whatever broke will break
3123
+ // the next file the same way.
3124
+ try {
3125
+ fs.renameSync(claimedPath, filePath);
3126
+ }
3127
+ catch { /* recovery sweeper will handle */ }
3128
+ throw e;
3129
+ }
3130
+ }
3131
+ }
3132
+ finally {
3133
+ try {
3134
+ await client.logout();
3135
+ }
3136
+ catch { /* ignore */ }
3137
+ }
3138
+ }
3139
+ catch {
3140
+ // IMAP still unreachable — leave files for next attempt
3141
+ }
3142
+ }
3143
+ /** Send a raw RFC 2822 message via SMTP for a given account.
3144
+ * Uses @bobfrankston/smtp-direct with the same TransportFactory as IMAP —
3145
+ * same TCP byte-stream interface, no nodemailer dependency. */
3146
+ async sendRawViaSMTP(accountId, raw) {
3147
+ const settings = loadSettings();
3148
+ const account = settings.accounts.find(a => a.id === accountId);
3149
+ if (!account?.smtp)
3150
+ throw new Error(`No SMTP config for ${accountId}`);
3151
+ const smtpPort = account.smtp.port || SMTP_PORT_STARTTLS;
3152
+ const smtpHost = account.smtp.host || account.imap?.host;
3153
+ if (!smtpHost)
3154
+ throw new Error(`No SMTP host for ${accountId}`);
3155
+ // SMTP auth: explicit SMTP creds, fall back to IMAP creds for password auth.
3156
+ const smtpAuthType = account.smtp.auth || (account.imap?.password ? "password" : undefined);
3157
+ const smtpUser = account.smtp.user || account.imap?.user || account.email;
3158
+ let auth;
3159
+ if (smtpAuthType === "password") {
3160
+ const pass = account.smtp.password || account.imap?.password;
3161
+ if (!pass)
3162
+ throw new Error("SMTP password not configured");
3163
+ auth = { method: "PLAIN", user: smtpUser, pass };
3164
+ }
3165
+ else if (smtpAuthType === "oauth2") {
3166
+ const token = await this.getOAuthToken(accountId);
3167
+ if (!token)
3168
+ throw new Error("OAuth token not available");
3169
+ auth = { method: "XOAUTH2", user: smtpUser, token };
3170
+ }
3171
+ const parseAddrs = (s) => s.match(/[\w.+-]+@[\w.-]+/g) || [];
3172
+ const toMatch = raw.match(/^To:\s*(.+)$/mi);
3173
+ const ccMatch = raw.match(/^Cc:\s*(.+)$/mi);
3174
+ const bccMatch = raw.match(/^Bcc:\s*(.+)$/mi);
3175
+ const fromMatch = raw.match(/^From:\s*(.+)$/mi);
3176
+ const subjectMatch = raw.match(/^Subject:\s*(.+)$/mi);
3177
+ const messageIdMatch = raw.match(/^Message-ID:\s*(<[^>]+>)/mi);
3178
+ const recipients = [
3179
+ ...(toMatch ? parseAddrs(toMatch[1]) : []),
3180
+ ...(ccMatch ? parseAddrs(ccMatch[1]) : []),
3181
+ ...(bccMatch ? parseAddrs(bccMatch[1]) : []),
3182
+ ];
3183
+ const sender = fromMatch ? (parseAddrs(fromMatch[1])[0] || account.email) : account.email;
3184
+ if (recipients.length === 0)
3185
+ throw new Error("No recipients");
3186
+ // Dedup: skip if this Message-ID has already been sent. Prevents the
3187
+ // outbox from re-sending the same file across crash/restart cycles.
3188
+ const messageId = messageIdMatch ? messageIdMatch[1] : "";
3189
+ if (messageId && this.db.hasSentMessage(messageId)) {
3190
+ console.log(` [smtp] ${accountId}: SKIP ${messageId} — already in sent_log`);
3191
+ return;
3192
+ }
3193
+ const rawToSend = raw.replace(/^Bcc:.*\r?\n/mi, "");
3194
+ this.saveSendingCopy(accountId, rawToSend, "sent");
3195
+ const smtp = new SmtpClient({
3196
+ host: smtpHost,
3197
+ port: smtpPort,
3198
+ secure: smtpPort === SMTP_PORT_IMPLICIT_TLS,
3199
+ auth,
3200
+ localname: os.hostname(),
3201
+ }, this.transportFactory);
3202
+ try {
3203
+ await smtp.connect();
3204
+ const result = await smtp.sendMail({ from: sender, to: recipients }, rawToSend);
3205
+ if (result.rejected.length > 0) {
3206
+ console.log(` [smtp] ${accountId}: ${result.rejected.length} recipient(s) rejected: ${result.rejected.map(r => `${r.address} (${r.code})`).join(", ")}`);
3207
+ }
3208
+ }
3209
+ finally {
3210
+ try {
3211
+ await smtp.quit();
3212
+ }
3213
+ catch { /* ignore */ }
3214
+ }
3215
+ if (messageId) {
3216
+ this.db.recordSent(messageId, accountId, subjectMatch?.[1]?.trim() || "", recipients);
3217
+ }
3218
+ console.log(` [smtp] ${accountId}: sent to ${recipients.join(", ")}`);
3219
+ }
3220
+ /** Process Outbox — send pending messages with flag-based interlock.
3221
+ * Each per-UID step is its own withConnection({slow}) call so the queue
3222
+ * yields between messages: a click-to-view body in the middle of a
3223
+ * 10-message outbox drain doesn't wait for all 10 to finish. */
3224
+ async processOutbox(accountId) {
3225
+ const outboxFolder = this.findFolder(accountId, "outbox");
3226
+ if (!outboxFolder)
3227
+ return;
3228
+ // Gmail: skip IMAP outbox check — sending handled by processLocalQueue which sends directly via SMTP
3229
+ if (this.isGmailAccount(accountId))
3230
+ return;
3231
+ const settings = loadSettings();
3232
+ const account = settings.accounts.find(a => a.id === accountId);
3233
+ if (!account)
3234
+ return;
3235
+ // List UIDs first — quick command, fast lane.
3236
+ const uids = await this.withConnection(accountId, (client) => client.getUids(outboxFolder.path));
3237
+ if (uids.length === 0)
3238
+ return;
3239
+ const STALE_CLAIM_MS = 3600_000; // 1 hour — longer than any reasonable SMTP send
3240
+ const nowSec = Math.floor(Date.now() / 1000);
3241
+ const sendingFlag = `$Sending-${this.hostname}-${nowSec}`;
3242
+ const sentFolder = this.findFolder(accountId, "sent");
3243
+ for (const uid of uids) {
3244
+ // Each iteration is one slow-lane turn — fast-lane work can run
3245
+ // between iterations, so a body click during a long outbox drain
3246
+ // gets serviced promptly.
3247
+ const result = await this.withConnection(accountId, async (client) => {
3248
+ const flags = await client.getFlags(outboxFolder.path, uid);
3249
+ // Sweep stale claims. $Sending-<host>-<sec> with old timestamp,
3250
+ // or legacy $Sending-<host> without timestamp (treated as
3251
+ // stale; if the real owner is alive it'll re-claim next tick).
3252
+ const claimFlags = flags.filter((f) => f.startsWith("$Sending"));
3253
+ for (const cf of claimFlags) {
3254
+ const m = cf.match(/^\$Sending-(.+?)(?:-(\d+))?$/);
3255
+ if (!m)
3256
+ continue;
3257
+ const tsSec = m[2] ? parseInt(m[2]) : 0;
3258
+ const ageSec = nowSec - tsSec;
3259
+ if (ageSec * 1000 > STALE_CLAIM_MS) {
3260
+ try {
3261
+ await client.removeFlags(outboxFolder.path, uid, [cf]);
3262
+ console.log(` [outbox] Swept stale claim ${cf} on UID ${uid} (age ${Math.round(ageSec / 60)}m)`);
3263
+ }
3264
+ catch { /* ignore */ }
3265
+ }
3266
+ }
3267
+ const flagsNow = (claimFlags.length > 0)
3268
+ ? await client.getFlags(outboxFolder.path, uid)
3269
+ : flags;
3270
+ if (flagsNow.some((f) => f.startsWith("$Sending")))
3271
+ return { skip: true };
3272
+ if (flagsNow.includes("$PermanentFailure"))
3273
+ return { skip: true };
3274
+ if (flagsNow.includes("$Failed")) {
3275
+ await client.removeFlags(outboxFolder.path, uid, ["$Failed"]);
3276
+ }
3277
+ await client.addFlags(outboxFolder.path, uid, [sendingFlag]);
3278
+ // TOCTOU re-check: if two devices addFlags concurrently both
3279
+ // see ≥2 sending flags. Fail-safe: both back off, next tick
3280
+ // one wins.
3281
+ const flagsAfter = await client.getFlags(outboxFolder.path, uid);
3282
+ const sendingFlags = flagsAfter.filter((f) => f.startsWith("$Sending"));
3283
+ if (sendingFlags.length > 1 || (sendingFlags.length === 1 && sendingFlags[0] !== sendingFlag)) {
3284
+ await client.removeFlags(outboxFolder.path, uid, [sendingFlag]);
3285
+ return { skip: true };
3286
+ }
3287
+ const msg = await client.fetchMessageByUid(outboxFolder.path, uid, { source: true });
3288
+ if (!msg?.source) {
3289
+ await client.removeFlags(outboxFolder.path, uid, [sendingFlag]);
3290
+ return { skip: true };
3291
+ }
3292
+ return { source: msg.source };
3293
+ }, { slow: true });
3294
+ if (result.skip)
3295
+ continue;
3296
+ const source = result.source;
3297
+ // SMTP send is its own connection — not an IMAP op, doesn't go
3298
+ // through withConnection.
3299
+ try {
3300
+ await this.sendRawViaSMTP(accountId, source);
3301
+ console.log(` [outbox] Sent UID ${uid}`);
3302
+ // Delete from Outbox + copy to Sent. Done in two separate
3303
+ // withConnection calls so other work can interleave.
3304
+ await this.withConnection(accountId, async (client) => {
3305
+ // Delete FIRST to prevent double-send if Sent-copy fails.
3306
+ await client.deleteMessageByUid(outboxFolder.path, uid);
3307
+ }, { slow: true });
3308
+ if (sentFolder) {
3309
+ try {
3310
+ await this.withConnection(accountId, async (client) => {
3311
+ await client.appendMessage(sentFolder.path, source, ["\\Seen"]);
3312
+ }, { slow: true });
3313
+ this.syncFolder(accountId, sentFolder.id).catch(() => { });
3314
+ }
3315
+ catch (sentErr) {
3316
+ console.error(` [outbox] Failed to copy to Sent: ${sentErr.message} — message was sent successfully`);
3317
+ }
3318
+ this.syncFolder(accountId, outboxFolder.id).catch(() => { });
3319
+ }
3320
+ }
3321
+ catch (e) {
3322
+ const errMsg = e.message || String(e);
3323
+ console.error(` [outbox] Send failed UID ${uid}: ${errMsg}`);
3324
+ try {
3325
+ await this.withConnection(accountId, async (client) => {
3326
+ await client.removeFlags(outboxFolder.path, uid, [sendingFlag]);
3327
+ await client.addFlags(outboxFolder.path, uid, ["$Failed"]);
3328
+ }, { slow: true });
3329
+ }
3330
+ catch { /* best-effort */ }
3331
+ this.emit("accountError", accountId, `Send failed: ${errMsg}`, "Message kept in Outbox", false);
3332
+ if (/auth|login|credential|invalid/i.test(errMsg)) {
3333
+ this.outboxBackoff.set(accountId, Date.now() + 3600000); // 1 hour
3334
+ console.error(` [outbox] Auth failure for ${accountId} — outbox paused for 1 hour`);
3335
+ }
3336
+ }
3337
+ }
3338
+ }
3339
+ /** Start background Outbox worker — runs immediately then every 10 seconds */
3340
+ outboxBackoff = new Map(); // accountId → next retry timestamp
3341
+ outboxBackoffDelay = new Map(); // accountId → current delay ms
3342
+ startOutboxWorker() {
3343
+ if (this.outboxInterval)
3344
+ return;
3345
+ const processAll = async () => {
3346
+ const now = Date.now();
3347
+ for (const [accountId] of this.configs) {
3348
+ // Skip accounts in backoff
3349
+ const retryAfter = this.outboxBackoff.get(accountId) || 0;
3350
+ if (now < retryAfter)
3351
+ continue;
3352
+ try {
3353
+ await this.processLocalQueue(accountId);
3354
+ await this.processOutbox(accountId);
3355
+ // Success — clear backoff
3356
+ this.outboxBackoff.delete(accountId);
3357
+ this.outboxBackoffDelay.delete(accountId);
3358
+ }
3359
+ catch (e) {
3360
+ // Stale-socket errors (Dovecot silently drops idle connections,
3361
+ // or the sync path timed out and destroyed the socket): force a
3362
+ // fresh ops client so the next tick doesn't keep hitting the same
3363
+ // dead socket. Without reconnectOps, the dead client stays in the
3364
+ // opsClients map and every subsequent processOutbox call fails
3365
+ // immediately with "Not connected" — forever.
3366
+ const msg = String(e?.message || e);
3367
+ if (/Not connected|ECONNRESET|socket hang up|EPIPE|write after end/i.test(msg)) {
3368
+ this.reconnectOps(accountId).catch(() => { });
3369
+ console.error(` [outbox] Stale connection for ${accountId}: ${msg} — reconnecting`);
3370
+ }
3371
+ else {
3372
+ // Exponential backoff: 60s → 120s → 300s (max 5min)
3373
+ const prevDelay = this.outboxBackoffDelay.get(accountId) || 0;
3374
+ const delay = prevDelay ? Math.min(prevDelay * 2, 300000) : 60000;
3375
+ this.outboxBackoffDelay.set(accountId, delay);
3376
+ this.outboxBackoff.set(accountId, now + delay);
3377
+ console.error(` [outbox] Error for ${accountId}: ${imapError(e)} (retry in ${Math.round(delay / 1000)}s)`);
3378
+ }
3379
+ }
3380
+ }
3381
+ // After each full tick, refresh the UI indicator.
3382
+ this.emitOutboxStatus();
3383
+ };
3384
+ // First tick at 500ms so any stale .sending-HOST-DEAD_PID claim file
3385
+ // left behind by a prior crash gets recovered (renamed back to .ltr)
3386
+ // within half a second of startup — otherwise the status-queue pill
3387
+ // shows a red "1 queued" to the user until the first 10s tick passes.
3388
+ setTimeout(() => processAll(), 500);
3389
+ this.outboxInterval = setInterval(processAll, 10000);
3390
+ }
3391
+ /** Stop Outbox worker */
3392
+ stopOutboxWorker() {
3393
+ if (this.outboxInterval) {
3394
+ clearInterval(this.outboxInterval);
3395
+ this.outboxInterval = null;
3396
+ }
3397
+ }
3398
+ // ── Config file watcher ──
3399
+ configWatchers = [];
3400
+ cloudPollTimers = [];
3401
+ /** Watch the local config files for external changes. On change, emit
3402
+ * configChanged so the UI can show a "restart to apply" banner. Uses
3403
+ * a debounce to coalesce rapid writes from save tools. */
3404
+ watchConfigFiles() {
3405
+ const files = ["accounts.jsonc", "allowlist.jsonc", "clients.jsonc", "config.jsonc", "contacts.jsonc"];
3406
+ const configDir = getConfigDir();
3407
+ const debounce = new Map();
3408
+ // Cache the last-seen normalized content per file. fs.watch fires on
3409
+ // metadata-only events (atime, attrib change) AND on no-op rewrites
3410
+ // that land identical bytes — both would fire a spurious banner.
3411
+ // Compare after the debounce window and only emit on a real change.
3412
+ const normalize = (s) => s.replace(/^\uFEFF/, "").replace(/\r\n/g, "\n").replace(/[ \t\r\n]+$/, "");
3413
+ const lastContent = new Map();
3414
+ for (const filename of files) {
3415
+ const full = path.join(configDir, filename);
3416
+ if (!fs.existsSync(full))
3417
+ continue;
3418
+ try {
3419
+ lastContent.set(filename, normalize(fs.readFileSync(full, "utf-8")));
3420
+ }
3421
+ catch { /* */ }
3422
+ try {
3423
+ const watcher = fs.watch(full, () => {
3424
+ const prev = debounce.get(filename);
3425
+ if (prev)
3426
+ clearTimeout(prev);
3427
+ debounce.set(filename, setTimeout(() => {
3428
+ debounce.delete(filename);
3429
+ let current = "";
3430
+ try {
3431
+ current = normalize(fs.readFileSync(full, "utf-8"));
3432
+ }
3433
+ catch { /* missing */ }
3434
+ const previous = lastContent.get(filename) || "";
3435
+ if (current === previous) {
3436
+ console.log(` [watch] ${filename} fs.watch fired but content unchanged — no banner`);
3437
+ return;
3438
+ }
3439
+ // Log a short diff hint so repeat-firings are diagnosable.
3440
+ const prevSize = previous.length;
3441
+ const curSize = current.length;
3442
+ const firstDiff = (() => {
3443
+ const n = Math.min(prevSize, curSize);
3444
+ for (let i = 0; i < n; i++)
3445
+ if (previous[i] !== current[i])
3446
+ return i;
3447
+ return n;
3448
+ })();
3449
+ const prevSnip = previous.slice(Math.max(0, firstDiff - 20), firstDiff + 40).replace(/\n/g, "\\n");
3450
+ const curSnip = current.slice(Math.max(0, firstDiff - 20), firstDiff + 40).replace(/\n/g, "\\n");
3451
+ console.log(` [watch] ${filename} changed: size ${prevSize}→${curSize}, first diff at byte ${firstDiff}`);
3452
+ console.log(` [watch] was: ${JSON.stringify(prevSnip)}`);
3453
+ console.log(` [watch] now: ${JSON.stringify(curSnip)}`);
3454
+ lastContent.set(filename, current);
3455
+ this.emit("configChanged", filename);
3456
+ }, 500));
3457
+ });
3458
+ this.configWatchers.push(watcher);
3459
+ }
3460
+ catch (e) {
3461
+ console.error(` [watch] Failed to watch ${filename}: ${e.message}`);
3462
+ }
3463
+ }
3464
+ // GDrive has no push/watch for arbitrary Drive files, so edits on
3465
+ // another device (or via Drive web) never fire fs.watch locally.
3466
+ // Poll the cloud copies of the replicated-to-cloud config files
3467
+ // (accounts.jsonc, allowlist.jsonc, clients.jsonc) every 3 minutes,
3468
+ // compare to local, and write-through on difference. The local
3469
+ // fs.watch above then picks up the write and emits configChanged.
3470
+ // config.jsonc is per-machine / local-only — never polled.
3471
+ const cloudFiles = ["accounts.jsonc", "allowlist.jsonc", "clients.jsonc", "contacts.jsonc"];
3472
+ const CLOUD_POLL_MS = 3 * 60 * 1000;
3473
+ // normalize() reused from the fs.watch block above — same intent:
3474
+ // cloud round-trips that re-wrap newlines / add a trailing newline are
3475
+ // semantically identical; don't overwrite local on those.
3476
+ const pollCloud = async () => {
3477
+ let cloudRead;
3478
+ let parseJsonc;
3479
+ try {
3480
+ ({ cloudRead } = await import("@bobfrankston/mailx-settings"));
3481
+ ({ parseJsonc } = await import("jsonc-parser").then(m => ({ parseJsonc: m.parse })));
3482
+ }
3483
+ catch {
3484
+ return; /* cloud module unavailable */
3485
+ }
3486
+ for (const filename of cloudFiles) {
3487
+ try {
3488
+ const cloudContent = await cloudRead(filename);
3489
+ if (!cloudContent)
3490
+ continue;
3491
+ const localPath = path.join(configDir, filename);
3492
+ let localContent = null;
3493
+ try {
3494
+ localContent = fs.readFileSync(localPath, "utf-8");
3495
+ }
3496
+ catch { /* missing */ }
3497
+ if (localContent !== null) {
3498
+ if (normalize(localContent) === normalize(cloudContent))
3499
+ continue;
3500
+ // Semantic check: parse both as JSONC and compare structures.
3501
+ // Catches reorderings that normalize() doesn't (e.g. JSON with
3502
+ // same keys in different order after a cloud-side re-serialize).
3503
+ try {
3504
+ const a = parseJsonc(localContent);
3505
+ const b = parseJsonc(cloudContent);
3506
+ if (a !== undefined && b !== undefined &&
3507
+ JSON.stringify(a) === JSON.stringify(b))
3508
+ continue;
3509
+ }
3510
+ catch { /* fall through to write */ }
3511
+ }
3512
+ fs.writeFileSync(localPath, cloudContent);
3513
+ console.log(` [cloud-poll] ${filename} updated from cloud copy`);
3514
+ }
3515
+ catch (e) {
3516
+ console.log(` [cloud-poll] ${filename} check skipped: ${e?.message || e}`);
3517
+ }
3518
+ }
3519
+ };
3520
+ // First poll ~10s after startup, then every 3 min.
3521
+ setTimeout(() => {
3522
+ pollCloud();
3523
+ const interval = setInterval(pollCloud, CLOUD_POLL_MS);
3524
+ this.cloudPollTimers.push(interval);
3525
+ }, 10_000);
3526
+ }
3527
+ /** Stop all config file watchers */
3528
+ stopWatchingConfig() {
3529
+ for (const w of this.configWatchers) {
3530
+ try {
3531
+ w.close();
3532
+ }
3533
+ catch { /* ignore */ }
3534
+ }
3535
+ this.configWatchers = [];
3536
+ for (const t of this.cloudPollTimers) {
3537
+ try {
3538
+ clearInterval(t);
3539
+ }
3540
+ catch { /* ignore */ }
3541
+ }
3542
+ this.cloudPollTimers = [];
3543
+ }
3544
+ // ── Google Contacts Sync (incremental via People API syncToken) ──
3545
+ /** Per-account in-flight guard so concurrent calls (startup + periodic
3546
+ * timer + manual button) share one round-trip instead of stacking. */
3547
+ contactsSyncing = new Map();
3548
+ /** Get an OAuth token for Google APIs (contacts, calendar, etc.)
3549
+ * Uses the SAME token as IMAP — scopes are combined in one grant */
3550
+ async getContactsToken(accountId) {
3551
+ // Reuse the IMAP token — it now includes contacts.readonly scope
3552
+ return this.getOAuthToken(accountId);
3553
+ }
3554
+ /** Sync contacts from Google People API. Incremental: persists
3555
+ * `nextSyncToken` per account in the kv table (`scope='contacts'`,
3556
+ * `key=accountId`) so subsequent calls only fetch changed/deleted rows.
3557
+ * First-ever call passes `requestSyncToken=true` so Google returns a
3558
+ * token to use next time; without that, the first response has no
3559
+ * `nextSyncToken` and incremental never kicks in. Returns the number of
3560
+ * contacts added or removed in this run. */
3561
+ async syncGoogleContacts(accountId) {
3562
+ // Coalesce concurrent calls for the same account.
3563
+ const inFlight = this.contactsSyncing.get(accountId);
3564
+ if (inFlight)
3565
+ return inFlight;
3566
+ const promise = this.syncGoogleContactsImpl(accountId)
3567
+ .finally(() => this.contactsSyncing.delete(accountId));
3568
+ this.contactsSyncing.set(accountId, promise);
3569
+ return promise;
3570
+ }
3571
+ async syncGoogleContactsImpl(accountId) {
3572
+ const token = await this.getContactsToken(accountId);
3573
+ if (!token)
3574
+ return 0;
3575
+ let changed = 0;
3576
+ let nextPageToken;
3577
+ const now = Date.now();
3578
+ // Per-account persisted sync token (survives restarts). Empty means
3579
+ // we've never completed an initial sync for this account.
3580
+ let syncToken = this.db.getKv("contacts", accountId) || "";
3581
+ try {
3582
+ do {
3583
+ const params = new URLSearchParams({
3584
+ personFields: "names,emailAddresses,organizations,photos",
3585
+ pageSize: "100",
3586
+ });
3587
+ if (nextPageToken)
3588
+ params.set("pageToken", nextPageToken);
3589
+ if (syncToken) {
3590
+ params.set("syncToken", syncToken);
3591
+ }
3592
+ else {
3593
+ // First-ever sync for this account — ask Google to give us
3594
+ // a token in the response so the NEXT call can be cheap.
3595
+ params.set("requestSyncToken", "true");
3596
+ }
3597
+ const url = `https://people.googleapis.com/v1/people/me/connections?${params}`;
3598
+ const res = await fetch(url, {
3599
+ headers: { Authorization: `Bearer ${token}` },
3600
+ });
3601
+ if (res.status === 410) {
3602
+ // Sync token expired (Google retains tokens for ~7 days).
3603
+ // Drop the stored token and recurse for a full sync — the
3604
+ // in-flight guard is on syncGoogleContacts (the public
3605
+ // wrapper), not Impl, so recursion is safe.
3606
+ this.db.setKv("contacts", accountId, null);
3607
+ return this.syncGoogleContactsImpl(accountId);
3608
+ }
3609
+ if (!res.ok) {
3610
+ const err = await res.text();
3611
+ console.error(` [contacts] API error for ${accountId}: ${res.status} ${err}`);
3612
+ return changed;
3613
+ }
3614
+ const data = await res.json();
3615
+ if (data.connections) {
3616
+ for (const person of data.connections) {
3617
+ const googleId = person.resourceName || "";
3618
+ // Incremental responses tag deleted contacts via
3619
+ // metadata.deleted=true (and emit no other fields).
3620
+ // Drop those rows so autocomplete stops surfacing them.
3621
+ if (person.metadata?.deleted) {
3622
+ const removed = this.db.deleteContactByGoogleId(googleId);
3623
+ if (removed > 0)
3624
+ changed += removed;
3625
+ continue;
3626
+ }
3627
+ const emails = person.emailAddresses || [];
3628
+ const names = person.names || [];
3629
+ const orgs = person.organizations || [];
3630
+ const name = names[0]?.displayName || "";
3631
+ const org = orgs[0]?.name || "";
3632
+ for (const emailEntry of emails) {
3633
+ const email = emailEntry.value?.toLowerCase();
3634
+ if (!email)
3635
+ continue;
3636
+ const existing = this.db.searchContacts(email, 1);
3637
+ const wasNew = !(existing.length > 0 && existing[0].email === email);
3638
+ this.db.recordSentAddress(name, email);
3639
+ if (wasNew)
3640
+ changed++;
3641
+ try {
3642
+ this.db.db.prepare("UPDATE contacts SET source = 'google', google_id = ?, organization = ?, updated_at = ? WHERE email = ?").run(googleId, org, now, email);
3643
+ }
3644
+ catch { /* ignore */ }
3645
+ }
3646
+ }
3647
+ }
3648
+ nextPageToken = data.nextPageToken;
3649
+ // Google only returns nextSyncToken on the LAST page of a
3650
+ // sync run (sentinel that the snapshot is consistent). When
3651
+ // it appears, persist it for next time.
3652
+ if (data.nextSyncToken) {
3653
+ this.db.setKv("contacts", accountId, data.nextSyncToken);
3654
+ syncToken = data.nextSyncToken;
3655
+ }
3656
+ } while (nextPageToken);
3657
+ console.log(` [contacts] ${accountId}: ${changed} change(s) (${syncToken ? "incremental" : "full"})`);
3658
+ }
3659
+ catch (e) {
3660
+ console.error(` [contacts] Sync error for ${accountId}: ${e.message}`);
3661
+ }
3662
+ return changed;
3663
+ }
3664
+ /** Sync contacts for all OAuth accounts */
3665
+ async syncAllContacts() {
3666
+ const settings = loadSettings();
3667
+ for (const account of settings.accounts) {
3668
+ if (account.imap.auth === "oauth2" && (account.enabled || account.syncContacts)) {
3669
+ await this.syncGoogleContacts(account.id);
3670
+ }
3671
+ }
3672
+ }
3673
+ /** Shut down all watchers and timers */
3674
+ async shutdown() {
3675
+ this.stopPeriodicSync();
3676
+ this.stopOutboxWorker();
3677
+ await this.stopWatching();
3678
+ // Disconnect all persistent operational connections
3679
+ for (const [accountId] of this.opsClients) {
3680
+ await this.disconnectOps(accountId);
3681
+ }
3682
+ }
3683
+ }
3684
+ //# sourceMappingURL=index.js.map