@bobfrankston/mailx-imap 0.1.99 → 0.1.102
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +34 -1
- package/index.js +173 -105
- package/package.json +3 -3
package/index.d.ts
CHANGED
|
@@ -271,6 +271,26 @@ export declare class ImapManager extends EventEmitter {
|
|
|
271
271
|
* the first caller finishes — important for outbox flushes that
|
|
272
272
|
* expect their syncFolder for `Sent` to actually run. */
|
|
273
273
|
private syncFolderLocks;
|
|
274
|
+
/** Per-folder timestamp of the last set-difference reconcile. Drives the
|
|
275
|
+
* throttle for the QRESYNC-path gap backfill (see fetchServerOnlyUids /
|
|
276
|
+
* syncFolder QRESYNC branch). Empty at boot, so the first sync of each
|
|
277
|
+
* folder after a restart always runs a reconcile — which is exactly when
|
|
278
|
+
* a daemon-was-down gap needs to be healed. */
|
|
279
|
+
private lastReconcileMs;
|
|
280
|
+
/**
|
|
281
|
+
* Set-difference backfill. Fetch every server UID (within the history
|
|
282
|
+
* window) that we don't already have locally and aren't fetching this
|
|
283
|
+
* cycle. This is the ONLY mechanism that recovers a message which is on
|
|
284
|
+
* the server but missing locally — the high-water-mark fetch
|
|
285
|
+
* (fetchMessagesSinceUid) cannot, because it only looks above the local
|
|
286
|
+
* highest UID. Gaps it heals: a message that arrived while the daemon was
|
|
287
|
+
* down (so it sits below the watermark once the next poll advances it), or
|
|
288
|
+
* one dropped mid-stream by a response desync. Returns the recovered
|
|
289
|
+
* messages (caller stores them) plus the server UID list it fetched, so the
|
|
290
|
+
* caller's deletion-reconcile can reuse it instead of paying for a second
|
|
291
|
+
* UID SEARCH round-trip.
|
|
292
|
+
*/
|
|
293
|
+
private fetchServerOnlyUids;
|
|
274
294
|
syncFolder(accountId: string, folderId: number, client?: any): Promise<number>;
|
|
275
295
|
private _syncFolderImpl;
|
|
276
296
|
/** Sync all folders for all accounts */
|
|
@@ -354,6 +374,19 @@ export declare class ImapManager extends EventEmitter {
|
|
|
354
374
|
* before _Spam / Sent / archive folders had a chance to rebind. */
|
|
355
375
|
private static readonly RECONCILE_DELETE_GRACE_MS;
|
|
356
376
|
private scheduleDeferredReconcileDelete;
|
|
377
|
+
/** Delete local rows whose uid >= the server's UIDNEXT for this folder.
|
|
378
|
+
* Such a uid was NEVER assigned in this folder (UIDNEXT is the next one the
|
|
379
|
+
* server will ever hand out), so the row is a cross-wired phantom ("UID is
|
|
380
|
+
* not identity" corruption) — a real message stamped with the wrong
|
|
381
|
+
* folder_id. They break prefetch (the server returns 0 FETCH responses for
|
|
382
|
+
* uids it doesn't have → no body caches) and the normal set-diff reconcile
|
|
383
|
+
* refuses to remove them when they're >50% of the folder. Deleting by the
|
|
384
|
+
* UIDNEXT invariant is unconditional-safe: the uid is impossible, can't be a
|
|
385
|
+
* move target, and the message's real copy re-syncs from its true folder
|
|
386
|
+
* (the local store is a cache). Caller must have a TRUSTWORTHY uidNext
|
|
387
|
+
* (STATUS result) AND confirmed UIDVALIDITY is unchanged for this folder
|
|
388
|
+
* (full-sync and QRESYNC-success paths both satisfy that). Returns the count. */
|
|
389
|
+
private sweepPhantomRows;
|
|
357
390
|
private unlinkBodyFile;
|
|
358
391
|
/** Fetch a single message body on demand, caching in the store.
|
|
359
392
|
*
|
|
@@ -366,7 +399,7 @@ export declare class ImapManager extends EventEmitter {
|
|
|
366
399
|
* Server fetch goes through the unified ops queue on the fast lane —
|
|
367
400
|
* the user clicked, they're waiting, this jumps ahead of any background
|
|
368
401
|
* prefetch sitting in the slow lane. */
|
|
369
|
-
fetchMessageBody(accountId: string, folderId: number, uid: number): Promise<Buffer | null>;
|
|
402
|
+
fetchMessageBody(accountId: string, folderId: number, uid: number, force?: boolean): Promise<Buffer | null>;
|
|
370
403
|
/** Fetch message body via Gmail/Outlook API.
|
|
371
404
|
* Throws `MessageNotFoundError` when the server says the message is gone
|
|
372
405
|
* (deleted from another device, for example). The caller uses that to
|
package/index.js
CHANGED
|
@@ -21,6 +21,11 @@ const SMTP_PORT_IMPLICIT_TLS = 465;
|
|
|
21
21
|
* the same file is retried. Gives the server time to settle so a retry after a
|
|
22
22
|
* lost-ack doesn't arrive while the first copy is still being processed. */
|
|
23
23
|
const OUTBOX_RETRY_DELAY_MS = 60000;
|
|
24
|
+
/** How often the QRESYNC fast path runs a full set-difference reconcile to
|
|
25
|
+
* heal gaps the high-water-mark fetch can't see. The first sync of a folder
|
|
26
|
+
* after boot always runs one (the map is empty); thereafter it's throttled to
|
|
27
|
+
* this interval so the QRESYNC speedup is preserved in steady state. */
|
|
28
|
+
const RECONCILE_THROTTLE_MS = 15 * 60 * 1000;
|
|
24
29
|
/** Parse X-Mailx-Retry* tracking headers from a raw RFC822 message. */
|
|
25
30
|
function parseRetryInfo(raw) {
|
|
26
31
|
const headerEnd = raw.search(/\r?\n\r?\n/);
|
|
@@ -1307,6 +1312,77 @@ export class ImapManager extends EventEmitter {
|
|
|
1307
1312
|
* the first caller finishes — important for outbox flushes that
|
|
1308
1313
|
* expect their syncFolder for `Sent` to actually run. */
|
|
1309
1314
|
syncFolderLocks = new Map();
|
|
1315
|
+
/** Per-folder timestamp of the last set-difference reconcile. Drives the
|
|
1316
|
+
* throttle for the QRESYNC-path gap backfill (see fetchServerOnlyUids /
|
|
1317
|
+
* syncFolder QRESYNC branch). Empty at boot, so the first sync of each
|
|
1318
|
+
* folder after a restart always runs a reconcile — which is exactly when
|
|
1319
|
+
* a daemon-was-down gap needs to be healed. */
|
|
1320
|
+
lastReconcileMs = new Map();
|
|
1321
|
+
/**
|
|
1322
|
+
* Set-difference backfill. Fetch every server UID (within the history
|
|
1323
|
+
* window) that we don't already have locally and aren't fetching this
|
|
1324
|
+
* cycle. This is the ONLY mechanism that recovers a message which is on
|
|
1325
|
+
* the server but missing locally — the high-water-mark fetch
|
|
1326
|
+
* (fetchMessagesSinceUid) cannot, because it only looks above the local
|
|
1327
|
+
* highest UID. Gaps it heals: a message that arrived while the daemon was
|
|
1328
|
+
* down (so it sits below the watermark once the next poll advances it), or
|
|
1329
|
+
* one dropped mid-stream by a response desync. Returns the recovered
|
|
1330
|
+
* messages (caller stores them) plus the server UID list it fetched, so the
|
|
1331
|
+
* caller's deletion-reconcile can reuse it instead of paying for a second
|
|
1332
|
+
* UID SEARCH round-trip.
|
|
1333
|
+
*/
|
|
1334
|
+
async fetchServerOnlyUids(client, accountId, folderId, folderPath, statusMessageCount, excludeUids) {
|
|
1335
|
+
const existingUids = this.db.getUidsForFolder(accountId, folderId);
|
|
1336
|
+
// Small folders (Drafts, Sent, …) get a FULL UID fetch so server-side
|
|
1337
|
+
// deletions of OLDER messages are reflected; large folders (134k INBOX)
|
|
1338
|
+
// get a date-bounded query so we don't enumerate the whole mailbox.
|
|
1339
|
+
const SMALL_FOLDER_THRESHOLD = 500;
|
|
1340
|
+
const isSmallFolder = statusMessageCount !== null && statusMessageCount <= SMALL_FOLDER_THRESHOLD;
|
|
1341
|
+
const historyDays = getHistoryDays(accountId);
|
|
1342
|
+
const SINCE_DAYS = historyDays > 0 ? historyDays : 1825;
|
|
1343
|
+
const sinceDate = new Date(Date.now() - SINCE_DAYS * 86400000);
|
|
1344
|
+
let serverUids;
|
|
1345
|
+
let dateBounded;
|
|
1346
|
+
const __t0 = Date.now();
|
|
1347
|
+
if (isSmallFolder) {
|
|
1348
|
+
console.log(` [sync-uids] ${accountId}/${folderPath}: small folder (server=${statusMessageCount}) — getUids ALL`);
|
|
1349
|
+
serverUids = await client.getUids(folderPath);
|
|
1350
|
+
dateBounded = false;
|
|
1351
|
+
}
|
|
1352
|
+
else {
|
|
1353
|
+
console.log(` [sync-uids] ${accountId}/${folderPath}: getUidsSince ${sinceDate.toISOString().slice(0, 10)}...`);
|
|
1354
|
+
serverUids = typeof client.getUidsSince === "function"
|
|
1355
|
+
? await client.getUidsSince(folderPath, sinceDate)
|
|
1356
|
+
: await client.getUids(folderPath);
|
|
1357
|
+
dateBounded = true;
|
|
1358
|
+
}
|
|
1359
|
+
console.log(` [sync-uids] ${accountId}/${folderPath}: ${serverUids.length} UIDs in ${Date.now() - __t0}ms`);
|
|
1360
|
+
const existingSet = new Set(existingUids);
|
|
1361
|
+
const missingUids = serverUids.filter((uid) => !existingSet.has(uid) && !excludeUids.has(uid));
|
|
1362
|
+
const recovered = [];
|
|
1363
|
+
if (missingUids.length === 0)
|
|
1364
|
+
return { recovered, serverUids, dateBounded };
|
|
1365
|
+
// Cap so a misbehaving response / brand-new account doesn't pull tens
|
|
1366
|
+
// of thousands at once; the rest resumes next cycle. Higher UID ≈ more
|
|
1367
|
+
// recent under stable UIDVALIDITY, so prefer newest when capping.
|
|
1368
|
+
const BACKFILL_CHUNK_SIZE = 100;
|
|
1369
|
+
const RECONCILE_FETCH_CAP = 5000;
|
|
1370
|
+
let toFetch = missingUids;
|
|
1371
|
+
if (missingUids.length > RECONCILE_FETCH_CAP) {
|
|
1372
|
+
console.log(` ${folderPath}: ${missingUids.length} server-only UIDs — capped at ${RECONCILE_FETCH_CAP}; will resume next cycle`);
|
|
1373
|
+
toFetch = missingUids.slice().sort((a, b) => b - a).slice(0, RECONCILE_FETCH_CAP);
|
|
1374
|
+
}
|
|
1375
|
+
else {
|
|
1376
|
+
console.log(` ${folderPath}: ${missingUids.length} server-only UIDs — fetching`);
|
|
1377
|
+
}
|
|
1378
|
+
for (let i = 0; i < toFetch.length; i += BACKFILL_CHUNK_SIZE) {
|
|
1379
|
+
const chunk = toFetch.slice(i, i + BACKFILL_CHUNK_SIZE);
|
|
1380
|
+
const got = await client.fetchMessages(folderPath, chunk.join(","), { source: false });
|
|
1381
|
+
recovered.push(...got);
|
|
1382
|
+
console.log(` ${folderPath}: backfill ${recovered.length}/${toFetch.length}`);
|
|
1383
|
+
}
|
|
1384
|
+
return { recovered, serverUids, dateBounded };
|
|
1385
|
+
}
|
|
1310
1386
|
async syncFolder(accountId, folderId, client) {
|
|
1311
1387
|
const lockKey = `${accountId}:${folderId}`;
|
|
1312
1388
|
const inflight = this.syncFolderLocks.get(lockKey);
|
|
@@ -1501,13 +1577,50 @@ export class ImapManager extends EventEmitter {
|
|
|
1501
1577
|
await this.storeMessages(accountId, folderId, folder, newOnes, highestUid);
|
|
1502
1578
|
console.log(` [qr-phase] ${folder.path}: storeMessages(${newOnes.length}) in ${Date.now() - __t2}ms`);
|
|
1503
1579
|
}
|
|
1580
|
+
// SET-DIFF GAP RECOVERY (bounded + throttled). The
|
|
1581
|
+
// fetchMessagesSinceUid above is a high-water-mark fetch —
|
|
1582
|
+
// it CANNOT recover a server UID that's missing below our
|
|
1583
|
+
// highest (a message that arrived while the daemon was down,
|
|
1584
|
+
// now under the watermark the next poll advances) or one
|
|
1585
|
+
// dropped mid-stream by a response desync. QRESYNC otherwise
|
|
1586
|
+
// returns here, making such gaps permanent (Bob 2026-06-20:
|
|
1587
|
+
// "messages before 7am are missing" — daemon crashed
|
|
1588
|
+
// overnight; the catch-up since-fetch lost the low UIDs of
|
|
1589
|
+
// the range and QRESYNC never reconciled). Run a full
|
|
1590
|
+
// set-diff on the first sync after boot, then at most once
|
|
1591
|
+
// per RECONCILE_THROTTLE_MS so the fast path stays fast.
|
|
1592
|
+
let backfilledCount = 0;
|
|
1593
|
+
const lastRecon = this.lastReconcileMs.get(folderId) ?? 0;
|
|
1594
|
+
if (Date.now() - lastRecon > RECONCILE_THROTTLE_MS) {
|
|
1595
|
+
this.lastReconcileMs.set(folderId, Date.now());
|
|
1596
|
+
try {
|
|
1597
|
+
const sd = await this.fetchServerOnlyUids(client, accountId, folderId, folder.path, statusMessageCount, new Set(newOnes.map((m) => m.uid)));
|
|
1598
|
+
if (sd.recovered.length > 0) {
|
|
1599
|
+
backfilledCount = await this.storeMessages(accountId, folderId, folder, sd.recovered, highestUid);
|
|
1600
|
+
console.log(` [qresync] ${accountId}/${folder.path}: set-diff backfilled ${backfilledCount} server-only UID(s)`);
|
|
1601
|
+
this.db.recalcFolderCounts(folderId);
|
|
1602
|
+
this.emit("folderCountsChanged", accountId, {});
|
|
1603
|
+
}
|
|
1604
|
+
}
|
|
1605
|
+
catch (e) {
|
|
1606
|
+
console.error(` [qresync] ${accountId}/${folder.path}: set-diff backfill failed: ${e?.message || e}`);
|
|
1607
|
+
}
|
|
1608
|
+
}
|
|
1504
1609
|
// Persist new watermark — next resync starts here.
|
|
1505
1610
|
if (qr.newHighestModSeq !== undefined) {
|
|
1506
1611
|
this.db.updateFolderSync(folderId, qr.exists ? prevUidValidity : prevUidValidity, String(qr.newHighestModSeq));
|
|
1507
1612
|
}
|
|
1613
|
+
// QRESYNC confirmed UIDVALIDITY is unchanged (it falls back to
|
|
1614
|
+
// full sync otherwise), so serverUidNext is trustworthy — sweep
|
|
1615
|
+
// phantom (uid >= UIDNEXT) cross-wired rows here too. Without
|
|
1616
|
+
// this, folders with a stable modseq (most of them) take the
|
|
1617
|
+
// QRESYNC fast path forever and their phantoms never get swept
|
|
1618
|
+
// (Bob 2026-06-17: CEBog phantoms persisted through the
|
|
1619
|
+
// reconcile-only sweep because CEBog syncs via QRESYNC).
|
|
1620
|
+
this.sweepPhantomRows(accountId, folderId, folder.path, serverUidNext);
|
|
1508
1621
|
this.emit("syncProgress", accountId, `sync:${folder.path}`, 100);
|
|
1509
1622
|
console.log(` [qresync] ${accountId}/${folder.path}: done in ${Date.now() - __sfStart}ms (path: QRESYNC)`);
|
|
1510
|
-
return newOnes.length;
|
|
1623
|
+
return newOnes.length + backfilledCount;
|
|
1511
1624
|
}
|
|
1512
1625
|
}
|
|
1513
1626
|
catch (qrErr) {
|
|
@@ -1563,96 +1676,22 @@ export class ImapManager extends EventEmitter {
|
|
|
1563
1676
|
// brand-new account doesn't try to pull tens of thousands of
|
|
1564
1677
|
// historical messages in one cycle. Hit the cap and the next
|
|
1565
1678
|
// sync picks up the rest.
|
|
1566
|
-
const existingUids = this.db.getUidsForFolder(accountId, folderId);
|
|
1567
1679
|
try {
|
|
1568
|
-
//
|
|
1569
|
-
//
|
|
1570
|
-
//
|
|
1571
|
-
//
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
//
|
|
1575
|
-
//
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
// anyway).
|
|
1579
|
-
// Small folders (Drafts, Sent, Outbox, Trash, …) get a FULL
|
|
1580
|
-
// UID fetch instead of date-bounded. Without this, server-side
|
|
1581
|
-
// deletions of OLDER messages (e.g., user emptied Trash from
|
|
1582
|
-
// Thunderbird) are never reflected — date-bound filters them
|
|
1583
|
-
// out of the comparison entirely. Bob 2026-05-12: "I deleted
|
|
1584
|
-
// my drafts and Thunderbird shows that but rmfmail is not
|
|
1585
|
-
// acknowledging the deletions." Threshold tuned conservatively
|
|
1586
|
-
// — UID SEARCH ALL on a 500-message folder is sub-second; the
|
|
1587
|
-
// pathology being avoided is the same call on 130k+ INBOX.
|
|
1588
|
-
const SMALL_FOLDER_THRESHOLD = 500;
|
|
1589
|
-
const isSmallFolder = statusMessageCount !== null && statusMessageCount <= SMALL_FOLDER_THRESHOLD;
|
|
1590
|
-
const SINCE_DAYS = effectiveDays > 0 ? effectiveDays : 1825;
|
|
1591
|
-
const sinceDate = new Date(Date.now() - SINCE_DAYS * 86400000);
|
|
1592
|
-
let allServerUids;
|
|
1593
|
-
const __uidsT0 = Date.now();
|
|
1594
|
-
if (isSmallFolder) {
|
|
1595
|
-
console.log(` [sync-uids] ${accountId}/${folder.path}: small folder (server=${statusMessageCount}) — getUids ALL`);
|
|
1596
|
-
allServerUids = await client.getUids(folder.path);
|
|
1597
|
-
serverUidsAreDateBounded = false;
|
|
1598
|
-
}
|
|
1599
|
-
else {
|
|
1600
|
-
console.log(` [sync-uids] ${accountId}/${folder.path}: getUidsSince ${sinceDate.toISOString().slice(0, 10)}...`);
|
|
1601
|
-
allServerUids = typeof client.getUidsSince === "function"
|
|
1602
|
-
? await client.getUidsSince(folder.path, sinceDate)
|
|
1603
|
-
: await client.getUids(folder.path);
|
|
1604
|
-
serverUidsAreDateBounded = true;
|
|
1605
|
-
}
|
|
1606
|
-
console.log(` [sync-uids] ${accountId}/${folder.path}: ${allServerUids.length} UIDs in ${Date.now() - __uidsT0}ms`);
|
|
1607
|
-
// Stash for the deletion-reconciliation block below — we
|
|
1608
|
-
// already have the server UID list, no point hitting the
|
|
1609
|
-
// server a second time.
|
|
1610
|
-
serverUidsCached = allServerUids;
|
|
1611
|
-
const existingSet = new Set(existingUids);
|
|
1612
|
-
const newSet = new Set(messages.map(m => m.uid));
|
|
1613
|
-
const missingUids = allServerUids.filter((uid) => !existingSet.has(uid) && !newSet.has(uid));
|
|
1614
|
-
// Backfill chunk size. Use the passed-in `client` directly
|
|
1615
|
-
// (NOT a nested withConnection) — syncFolder is now wrapped
|
|
1616
|
-
// in withConnection at the call site, so the slow lane is
|
|
1617
|
-
// already locked for our duration. A nested withConnection
|
|
1618
|
-
// would deadlock waiting for the slot we hold.
|
|
1619
|
-
const BACKFILL_CHUNK_SIZE = 100;
|
|
1620
|
-
if (missingUids.length > 0 && missingUids.length <= 5000) {
|
|
1621
|
-
let minU = existingUids[0] ?? 0;
|
|
1622
|
-
for (let i = 1; i < existingUids.length; i++)
|
|
1623
|
-
if (existingUids[i] < minU)
|
|
1624
|
-
minU = existingUids[i];
|
|
1625
|
-
console.log(` ${folder.path}: ${missingUids.length} server-only UIDs (local lowest=${minU}, highest=${highestUid}) — fetching`);
|
|
1626
|
-
let recoveredTotal = 0;
|
|
1627
|
-
for (let i = 0; i < missingUids.length; i += BACKFILL_CHUNK_SIZE) {
|
|
1628
|
-
const chunk = missingUids.slice(i, i + BACKFILL_CHUNK_SIZE);
|
|
1629
|
-
const range = chunk.join(",");
|
|
1630
|
-
const recovered = await client.fetchMessages(folder.path, range, { source: false });
|
|
1631
|
-
messages.push(...recovered);
|
|
1632
|
-
recoveredTotal += recovered.length;
|
|
1633
|
-
console.log(` ${folder.path}: fetch ${recoveredTotal}/${missingUids.length}`);
|
|
1634
|
-
}
|
|
1635
|
-
}
|
|
1636
|
-
else if (missingUids.length > 5000) {
|
|
1637
|
-
console.log(` ${folder.path}: ${missingUids.length} server-only UIDs — capped; will resume next cycle`);
|
|
1638
|
-
// Vanilla IMAP under stable UIDVALIDITY: higher UID =
|
|
1639
|
-
// later assignment ≈ more recent message (Dovecot/
|
|
1640
|
-
// Cyrus). Gmail-API path is separate (no temporal
|
|
1641
|
-
// meaning to its hash-UIDs).
|
|
1642
|
-
const cappedSlice = missingUids.sort((a, b) => b - a).slice(0, 5000);
|
|
1643
|
-
let recoveredTotal = 0;
|
|
1644
|
-
for (let i = 0; i < cappedSlice.length; i += BACKFILL_CHUNK_SIZE) {
|
|
1645
|
-
const chunk = cappedSlice.slice(i, i + BACKFILL_CHUNK_SIZE);
|
|
1646
|
-
const recovered = await client.fetchMessages(folder.path, chunk.join(","), { source: false });
|
|
1647
|
-
messages.push(...recovered);
|
|
1648
|
-
recoveredTotal += recovered.length;
|
|
1649
|
-
console.log(` ${folder.path}: fetch ${recoveredTotal}/5000 (capped)`);
|
|
1650
|
-
}
|
|
1651
|
-
}
|
|
1680
|
+
// Shared with the QRESYNC fast path (fetchServerOnlyUids).
|
|
1681
|
+
// Exclude the UIDs we just fetched via fetchMessagesSinceUid —
|
|
1682
|
+
// they're in `messages` but not yet in the DB, so they'd
|
|
1683
|
+
// otherwise look "missing" and get re-fetched.
|
|
1684
|
+
const sd = await this.fetchServerOnlyUids(client, accountId, folderId, folder.path, statusMessageCount, new Set(messages.map(m => m.uid)));
|
|
1685
|
+
messages.push(...sd.recovered);
|
|
1686
|
+
// Stash the server UID list for the deletion-reconciliation
|
|
1687
|
+
// block below — no point hitting the server a second time.
|
|
1688
|
+
serverUidsCached = sd.serverUids;
|
|
1689
|
+
serverUidsAreDateBounded = sd.dateBounded;
|
|
1652
1690
|
}
|
|
1653
1691
|
catch (e) {
|
|
1654
1692
|
console.error(` ${folder.path}: reconciliation failed: ${e.message}`);
|
|
1655
1693
|
}
|
|
1694
|
+
this.lastReconcileMs.set(folderId, Date.now());
|
|
1656
1695
|
// Date-based backfill via SEARCH SINCE was here. Removed —
|
|
1657
1696
|
// set-diff above already covers the same case (any server UID
|
|
1658
1697
|
// not in our local set gets fetched, regardless of whether it's
|
|
@@ -1828,23 +1867,7 @@ export class ImapManager extends EventEmitter {
|
|
|
1828
1867
|
// server hiccup — so delete them unconditionally (the real copy lives
|
|
1829
1868
|
// in its true folder and re-syncs from there; the local store is a
|
|
1830
1869
|
// cache). Only runs when STATUS gave a trustworthy UIDNEXT.
|
|
1831
|
-
|
|
1832
|
-
try {
|
|
1833
|
-
const phantoms = this.db.getUidsForFolder(accountId, folderId).filter(u => u >= serverUidNext);
|
|
1834
|
-
if (phantoms.length > 0) {
|
|
1835
|
-
for (const uid of phantoms) {
|
|
1836
|
-
this.unlinkBodyFile(accountId, uid, folderId).catch(() => { });
|
|
1837
|
-
this.db.deleteMessage(accountId, folderId, uid, `phantom: uid>=server UIDNEXT ${serverUidNext}`, `mailx-imap phantom sweep (${folder.path})`);
|
|
1838
|
-
}
|
|
1839
|
-
deletedCount += phantoms.length;
|
|
1840
|
-
this.db.recalcFolderCounts(folderId);
|
|
1841
|
-
console.log(` [phantom-sweep] ${accountId}/${folder.path}: removed ${phantoms.length} cross-wired row(s) with uid >= UIDNEXT ${serverUidNext}`);
|
|
1842
|
-
}
|
|
1843
|
-
}
|
|
1844
|
-
catch (e) {
|
|
1845
|
-
console.error(` [phantom-sweep] ${accountId}/${folder.path}: ${e?.message || e}`);
|
|
1846
|
-
}
|
|
1847
|
-
}
|
|
1870
|
+
deletedCount += this.sweepPhantomRows(accountId, folderId, folder.path, serverUidNext);
|
|
1848
1871
|
try {
|
|
1849
1872
|
// Reuse the server UID list set-diff already fetched.
|
|
1850
1873
|
// Without this we made TWO `UID SEARCH` calls per folder
|
|
@@ -2973,6 +2996,38 @@ export class ImapManager extends EventEmitter {
|
|
|
2973
2996
|
}, ImapManager.RECONCILE_DELETE_GRACE_MS);
|
|
2974
2997
|
this.deferredDeletes.set(key, t);
|
|
2975
2998
|
}
|
|
2999
|
+
/** Delete local rows whose uid >= the server's UIDNEXT for this folder.
|
|
3000
|
+
* Such a uid was NEVER assigned in this folder (UIDNEXT is the next one the
|
|
3001
|
+
* server will ever hand out), so the row is a cross-wired phantom ("UID is
|
|
3002
|
+
* not identity" corruption) — a real message stamped with the wrong
|
|
3003
|
+
* folder_id. They break prefetch (the server returns 0 FETCH responses for
|
|
3004
|
+
* uids it doesn't have → no body caches) and the normal set-diff reconcile
|
|
3005
|
+
* refuses to remove them when they're >50% of the folder. Deleting by the
|
|
3006
|
+
* UIDNEXT invariant is unconditional-safe: the uid is impossible, can't be a
|
|
3007
|
+
* move target, and the message's real copy re-syncs from its true folder
|
|
3008
|
+
* (the local store is a cache). Caller must have a TRUSTWORTHY uidNext
|
|
3009
|
+
* (STATUS result) AND confirmed UIDVALIDITY is unchanged for this folder
|
|
3010
|
+
* (full-sync and QRESYNC-success paths both satisfy that). Returns the count. */
|
|
3011
|
+
sweepPhantomRows(accountId, folderId, folderPath, serverUidNext) {
|
|
3012
|
+
if (serverUidNext == null || serverUidNext <= 0)
|
|
3013
|
+
return 0;
|
|
3014
|
+
try {
|
|
3015
|
+
const phantoms = this.db.getUidsForFolder(accountId, folderId).filter(u => u >= serverUidNext);
|
|
3016
|
+
if (phantoms.length === 0)
|
|
3017
|
+
return 0;
|
|
3018
|
+
for (const uid of phantoms) {
|
|
3019
|
+
this.unlinkBodyFile(accountId, uid, folderId).catch(() => { });
|
|
3020
|
+
this.db.deleteMessage(accountId, folderId, uid, `phantom: uid>=server UIDNEXT ${serverUidNext}`, `mailx-imap phantom sweep (${folderPath})`);
|
|
3021
|
+
}
|
|
3022
|
+
this.db.recalcFolderCounts(folderId);
|
|
3023
|
+
console.log(` [phantom-sweep] ${accountId}/${folderPath}: removed ${phantoms.length} cross-wired row(s) with uid >= UIDNEXT ${serverUidNext}`);
|
|
3024
|
+
return phantoms.length;
|
|
3025
|
+
}
|
|
3026
|
+
catch (e) {
|
|
3027
|
+
console.error(` [phantom-sweep] ${accountId}/${folderPath}: ${e?.message || e}`);
|
|
3028
|
+
return 0;
|
|
3029
|
+
}
|
|
3030
|
+
}
|
|
2976
3031
|
async unlinkBodyFile(accountId, uid, folderId) {
|
|
2977
3032
|
try {
|
|
2978
3033
|
const row = this.db.getMessageByUid(accountId, uid, folderId);
|
|
@@ -2993,7 +3048,7 @@ export class ImapManager extends EventEmitter {
|
|
|
2993
3048
|
* Server fetch goes through the unified ops queue on the fast lane —
|
|
2994
3049
|
* the user clicked, they're waiting, this jumps ahead of any background
|
|
2995
3050
|
* prefetch sitting in the slow lane. */
|
|
2996
|
-
async fetchMessageBody(accountId, folderId, uid) {
|
|
3051
|
+
async fetchMessageBody(accountId, folderId, uid, force = false) {
|
|
2997
3052
|
// Belt-and-braces against `UID FETCH 0`. The IMAP server rejects it as
|
|
2998
3053
|
// BAD "Invalid uidset" and the connection slot is consumed for the
|
|
2999
3054
|
// round-trip. The enqueue path now guards too — this catches direct
|
|
@@ -3018,7 +3073,17 @@ export class ImapManager extends EventEmitter {
|
|
|
3018
3073
|
// delayed until the client timed out. The 5min→12h backoff (shared with
|
|
3019
3074
|
// prefetch via isPrefetchEmpty) stops the tight re-fetch loop; the body
|
|
3020
3075
|
// simply shows as unavailable until the backoff lapses and one retry runs.
|
|
3021
|
-
|
|
3076
|
+
// EXCEPT an interactive click (`force`): the user is staring at a blank
|
|
3077
|
+
// preview demanding THIS body NOW. Honoring a *background* prefetch
|
|
3078
|
+
// failure's backoff for an explicit user action is a local-first
|
|
3079
|
+
// violation — it left a real INBOX message (bobma uid 4967554,
|
|
3080
|
+
// 2026-06-19) showing an eternal "getting body" spinner for up to 12 h
|
|
3081
|
+
// because one earlier prefetch chunk returned 0 bodies (an iflow parser
|
|
3082
|
+
// miss, not a real expunge — siblings recovered on retry). A forced
|
|
3083
|
+
// single-UID fetch re-attempts immediately; if it genuinely fails it
|
|
3084
|
+
// re-arms the backoff below and the reconciler emits bodyFetchError so
|
|
3085
|
+
// the viewer shows a banner instead of spinning forever.
|
|
3086
|
+
if (!force && this.isPrefetchEmpty(accountId, folderId, uid))
|
|
3022
3087
|
return null;
|
|
3023
3088
|
const folder = this.db.getFolders(accountId).find(f => f.id === folderId);
|
|
3024
3089
|
if (!folder)
|
|
@@ -3043,6 +3108,9 @@ export class ImapManager extends EventEmitter {
|
|
|
3043
3108
|
return null;
|
|
3044
3109
|
const bodyPath = await this.bodyStore.putMessage(accountId, folderId, uid, raw);
|
|
3045
3110
|
this.db.updateBodyPath(accountId, folderId, uid, bodyPath);
|
|
3111
|
+
// A forced fetch that beat a stale backoff just healed the row;
|
|
3112
|
+
// drop the prefetch-empty record so it doesn't suppress the next tick.
|
|
3113
|
+
this.clearPrefetchEmpty(accountId, folderId, uid);
|
|
3046
3114
|
this.emit("bodyCached", accountId, uid);
|
|
3047
3115
|
return raw;
|
|
3048
3116
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bobfrankston/mailx-imap",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.102",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"types": "index.d.ts",
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
"dependencies": {
|
|
12
12
|
"@bobfrankston/mailx-types": "^0.1.19",
|
|
13
13
|
"@bobfrankston/mailx-settings": "^0.1.28",
|
|
14
|
-
"@bobfrankston/mailx-store": "^0.1.
|
|
14
|
+
"@bobfrankston/mailx-store": "^0.1.54",
|
|
15
15
|
"@bobfrankston/iflow-direct": "^0.1.55",
|
|
16
16
|
"@bobfrankston/tcp-transport": "^0.1.7",
|
|
17
17
|
"@bobfrankston/smtp-direct": "^0.1.9",
|
|
@@ -39,7 +39,7 @@
|
|
|
39
39
|
"dependencies": {
|
|
40
40
|
"@bobfrankston/mailx-types": "^0.1.19",
|
|
41
41
|
"@bobfrankston/mailx-settings": "^0.1.28",
|
|
42
|
-
"@bobfrankston/mailx-store": "^0.1.
|
|
42
|
+
"@bobfrankston/mailx-store": "^0.1.54",
|
|
43
43
|
"@bobfrankston/iflow-direct": "^0.1.55",
|
|
44
44
|
"@bobfrankston/tcp-transport": "^0.1.7",
|
|
45
45
|
"@bobfrankston/smtp-direct": "^0.1.9",
|