@nocobase/app-plugin-mail 1.0.0-beta.23 → 1.0.0-beta.25

package/dist/server/adapters/imap-smtp/adapter.js

@@ -5,9 +5,10 @@ import { randomUUID } from 'node:crypto';
 import { simpleParser } from 'mailparser';
 import { Readable, addAbortSignal } from 'node:stream';
 import { attachmentParts, bodyParts, readBody, readRawSource, rawComparable, metadataQuery, attachmentMetadata, MAX_HEADER_BYTES, } from './content.js';
-import { IMAP_SMTP_CAPABILITIES, MAX_IMAP_BODY_BYTES, MAX_IMAP_DETAIL_BODY_BYTES, MAX_IMAP_PROVIDER_ID_LENGTH, MAX_IMAP_RAW_COMPARE_BUDGET, MAX_IMAP_RAW_COMPARE_BYTES, MAX_IMAP_VERIFY_ATTEMPTS, MAX_IMAP_VERIFY_STATE_BYTES, } from './constants.js';
+import { IMAP_SMTP_CAPABILITIES, MAX_IMAP_BODY_BYTES, MAX_IMAP_DETAIL_BODY_BYTES, MAX_IMAP_PROVIDER_ID_LENGTH, MAX_IMAP_RAW_COMPARE_BUDGET, MAX_IMAP_RAW_COMPARE_BYTES, MAX_IMAP_VERIFY_ATTEMPTS, MAX_IMAP_VERIFY_STATE_BYTES, MAX_IMAP_RECHECK_IDENTITIES, IMAP_RECHECK_BATCH_SIZE, } from './constants.js';
 import { createImapClient, createSmtpTransport } from './connection.js';
-import { compressUids, encodeCopyLocator, encodeHistoryCursor, encodeMessageLocator, folderCursor, parseHistoryCursor, parseSyncCursor, rangeFor, requireAttachmentLocator, requireMessageLocator, syncCursor, } from './locators.js';
+import { ImapCommandMetrics, observeImapCommands } from './command-metrics.js';
+import { compressUids, matchingUids, subtractUids, encodeCopyLocator, encodeHistoryCursor, encodeMessageLocator, folderCursor, parseHistoryCursor, parseSyncCursor, rangeFor, requireAttachmentLocator, requireMessageLocator, syncCursor, } from './locators.js';
 import { classifyFolderType, folderTypeFromPath } from './folders.js';
 function classifyConfiguredFolder(mailbox, sentFolder) {
     if (sentFolder && mailbox.path === sentFolder)
@@ -24,10 +25,14 @@ export class ImapSmtpAdapter {
     locations;
     protocol;
     operation;
+    pool;
     identity;
     capabilities = IMAP_SMTP_CAPABILITIES;
     folderTypes = new Map();
     imapClient;
+    commandMetrics = new ImapCommandMetrics();
+    stopCommandObservation;
+    yieldReasons = new Set();
     smtpTransport;
     // Evidence and byte budget of the current step only; never reused across steps.
     copies = [];
@@ -43,7 +48,7 @@ export class ImapSmtpAdapter {
         smtp: createSmtpTransport,
         expungeUid,
         transferUid,
-    }), operation) {
+    }), operation, pool) {
         this.context = context;
         this.config = config;
         this.account = account;
@@ -51,6 +56,7 @@ export class ImapSmtpAdapter {
         this.locations = locations;
         this.protocol = protocol;
         this.operation = operation;
+        this.pool = pool;
         if (locations && locations.accountId !== account.id)
             throw new Error('The IMAP persistence belongs to another account.');
         if (operation && operation.accountId !== account.id)
@@ -80,13 +86,13 @@ export class ImapSmtpAdapter {
             throw new Error('An IMAP message operation cannot be reused for synchronization.');
         if (run.accountId !== this.account.id)
             throw new Error('The IMAP sync account changed.');
-        return new ImapSmtpAdapter(this.context, this.config, this.account, this.credential, this.locations, run, this.protocol);
+        return new ImapSmtpAdapter(this.context, this.config, this.account, this.credential, this.locations, run, this.protocol, undefined, this.pool);
     }
     async forMessage(messageId, providerMessageId, operation, move) {
         if (!this.locations)
             throw new Error('IMAP location persistence is not configured.');
         const bound = await this.locations.bindMessage(messageId, providerMessageId, operation, move);
-        return new ImapSmtpAdapter(this.context, this.config, this.account, this.credential, bound.persistence, undefined, this.protocol, bound.snapshot);
+        return new ImapSmtpAdapter(this.context, this.config, this.account, this.credential, bound.persistence, undefined, this.protocol, bound.snapshot, this.pool);
     }
     /** True when this instance records message copies in the account's location table. */
     get persistsLocations() {
@@ -107,6 +113,9 @@ export class ImapSmtpAdapter {
                 input.run.leaseToken !== this.syncRun.leaseToken))
             throw new Error('The IMAP sync operation snapshot changed.');
         const incomplete = this.verificationIncomplete;
+        const changeCursor = incomplete && input.changeCursor
+            ? parseSyncCursor(input.changeCursor)
+            : undefined;
         const evidence = {
             copies: this.copies.filter((copy) => copy.flagsOnly || ids.has(copy.providerMessageId)),
             changes: this.changes,
@@ -114,7 +123,19 @@ export class ImapSmtpAdapter {
         };
         this.beginStep();
         return this.locations.commitSyncStep(incomplete
-            ? { ...input, status: 'partial', createNextTask: false }
+            ? {
+                ...input,
+                status: 'partial',
+                createNextTask: false,
+                changeCursor: changeCursor &&
+                    syncCursor({
+                        ...changeCursor,
+                        verify: {
+                            excluded: changeCursor.verify?.excluded,
+                            rechecks: changeCursor.verify?.rechecks,
+                        },
+                    }),
+            }
             : input, evidence);
     }
     async listFolders(input) {
@@ -130,6 +151,7 @@ export class ImapSmtpAdapter {
             };
         }
         catch (error) {
+            this.closeImap();
             return { ok: false, error: classifyError(error, 'IMAP_LIST_FOLDERS') };
         }
     }
@@ -142,9 +164,17 @@ export class ImapSmtpAdapter {
                 throwIfAborted(signal);
                 folders[mailbox.path] = await this.folderState(mailbox.path, mailbox);
             }
-            return { ok: true, value: syncCursor({ version: 1, folders }) };
+            return {
+                ok: true,
+                value: syncCursor({
+                    version: 1,
+                    folders,
+                    receivedAfter: this.syncRun?.policy.receivedAfter,
+                }),
+            };
         }
         catch (error) {
+            this.closeImap();
             return {
                 ok: false,
                 error: classifyError(error, 'IMAP_SYNC_CURSOR'),
@@ -177,12 +207,21 @@ export class ImapSmtpAdapter {
                 : undefined;
             since?.setUTCHours(0, 0, 0, 0);
             let scannedRanges = 0;
+            let budgetStartedAt;
+            const checkNewMail = this.newMailChecker(parseSyncCursor(this.syncRun?.changeCursor ?? input.baselineCursor)
+                .folders, input.signal);
             while (folderIndex < folders.length &&
                 messages.length < limit &&
                 scannedRanges < 10) {
                 scannedRanges += 1;
                 throwIfAborted(input.signal);
                 const folder = folders[folderIndex];
+                const selectedFolderIndex = folderIndex;
+                // Top pages share this call's budget from the first top page onward.
+                // Continuation pages keep their throughput and existing push checks.
+                const topPage = upperUid === undefined && legacyOffset === undefined;
+                if (topPage)
+                    budgetStartedAt ??= Date.now();
                 const mailbox = await (await this.imap()).mailboxOpen(folder, {
                     readOnly: true,
                 });
@@ -240,7 +279,16 @@ export class ImapSmtpAdapter {
                         upperUid = start - 1;
                     }
                 }
-                for (const copy of await this.fetchMessages(folder, String(mailbox.uidValidity), selected, input.signal)) {
+                let yielded = false;
+                const fetched = await this.fetchMessages(folder, String(mailbox.uidValidity), selected, input.signal, async () => {
+                    yielded = await checkNewMail();
+                    if (!yielded && topPage && Date.now() - budgetStartedAt >= 2_000) {
+                        this.yieldReasons.add('budget');
+                        yielded = true;
+                    }
+                    return yielded;
+                });
+                for (const copy of fetched) {
                     const { receivedAt } = copy.message;
                     if (input.receivedAfter &&
                         receivedAt &&
@@ -249,10 +297,16 @@ export class ImapSmtpAdapter {
                     }
                     messages.push(copy);
                 }
+                if (yielded) {
+                    folderIndex = selectedFolderIndex;
+                    upperUid = fetched.at(-1).tuple.uid - 1;
+                }
                 if (upperUid !== undefined && upperUid < 1) {
                     folderIndex += 1;
                     upperUid = undefined;
                 }
+                if (yielded)
+                    break;
             }
             const nextCursor = folderIndex < folders.length
                 ? encodeHistoryCursor({ folderIndex, upperUid })
@@ -267,6 +321,7 @@ export class ImapSmtpAdapter {
             };
         }
         catch (error) {
+            this.closeImap();
             return { ok: false, error: classifyError(error, 'IMAP_LIST_MESSAGES') };
         }
     }
@@ -290,7 +345,12 @@ export class ImapSmtpAdapter {
                 };
             }
             const mailboxes = this.selectableMailboxes(await this.listMailboxes());
-            const nextFolders = {};
+            // Unvisited folders must retain their high-water marks when a completed
+            // folder's new copies are committed before the next folder is inspected.
+            const nextFolders = Object.fromEntries(mailboxes.flatMap((mailbox) => {
+                const old = previous.folders[mailbox.path];
+                return old ? [[mailbox.path, old]] : [];
+            }));
             const messages = [];
             let hasMore = false;
             const limit = Math.max(1, input.limit);
@@ -339,19 +399,31 @@ export class ImapSmtpAdapter {
                     uidValidity: current.uidValidity,
                     uidNext: nextUid,
                 };
+                if (messages.length > 0) {
+                    hasMore ||= mailbox !== mailboxes.at(-1);
+                    break;
+                }
             }
             let verify;
             if (this.locations) {
                 if (hasMore || messages.length > 0) {
-                    // New copies are committed first; verification restarts afterwards.
-                    verify = previous.verify && {
-                        attempts: previous.verify.attempts,
-                        excluded: previous.verify.excluded,
-                    };
+                    // Commit new copies first, retaining the last complete verification pass.
+                    verify = previous.verify && { ...previous.verify };
+                    hasMore = true;
+                }
+                else if (this.syncRun?.mode === 'initial' &&
+                    !this.syncRun.historyComplete) {
+                    // Catch up between history pages without spending verification attempts.
+                    verify = previous.verify;
+                }
+                else if (previous.verify?.rechecks?.length) {
+                    // Recheck only committed identities, without mixing partial observations
+                    // into a verification pass. The cursor saves progress after each folder.
+                    verify = await this.advanceExcludedRecheck(previous.verify, nextFolders, input.signal);
                     hasMore = true;
                 }
                 else {
-                    const result = await this.verifyFolders(mailboxes, nextFolders, previous.verify, input.signal);
+                    const result = await this.verifyFolders(mailboxes, nextFolders, previous.verify, input.signal, previous.receivedAfter ?? this.syncRun?.policy.receivedAfter);
                     if (result === 'generation-changed')
                         return failure('IMAP_SYNC_CURSOR_INVALID', 'Mailbox generation changed during verification.', 'provider', false);
                     messages.push(...this.verificationCopies);
@@ -359,6 +431,8 @@ export class ImapSmtpAdapter {
                     verify = result.verify;
                 }
             }
+            if (verify?.excluded && messages.length)
+                verify = this.enqueueExcludedRechecks(verify, messages);
             return {
                 ok: true,
                 value: {
@@ -367,6 +441,7 @@ export class ImapSmtpAdapter {
                     deletedProviderMessageIds: [],
                     nextCursor: syncCursor({
                         version: 1,
+                        receivedAfter: previous.receivedAfter ?? this.syncRun?.policy.receivedAfter,
                         folders: nextFolders,
                         ...(verify ? { verify } : {}),
                     }),
@@ -375,6 +450,7 @@ export class ImapSmtpAdapter {
             };
         }
         catch (error) {
+            this.closeImap();
             return { ok: false, error: classifyError(error, 'IMAP_LIST_CHANGES') };
         }
     }
@@ -413,6 +489,7 @@ export class ImapSmtpAdapter {
             };
         }
         catch (error) {
+            this.closeImap();
             return { ok: false, error: classifyError(error, 'IMAP_GET_MESSAGE') };
         }
     }
@@ -479,7 +556,10 @@ export class ImapSmtpAdapter {
             if (!download.content)
                 return failure('IMAP_ATTACHMENT_NOT_FOUND', 'IMAP attachment was not found.', 'provider', false);
             download.content.once('end', release);
-            download.content.once('error', release);
+            download.content.once('error', () => {
+                this.closeImap();
+                release?.();
+            });
             download.content.once('close', release);
             streamed = true;
             if (signal)
@@ -503,6 +583,7 @@ export class ImapSmtpAdapter {
             };
         }
         catch (error) {
+            this.closeImap();
             return { ok: false, error: classifyError(error, 'IMAP_GET_ATTACHMENT') };
         }
         finally {
@@ -599,6 +680,7 @@ export class ImapSmtpAdapter {
                     }
                 }
                 catch (error) {
+                    this.closeImap();
                     // SMTP already accepted the message; never report this as a send failure.
                     sentCopyError = {
                         ...classifyError(error, 'IMAP_SAVE_SENT_COPY'),
@@ -616,6 +698,7 @@ export class ImapSmtpAdapter {
             };
         }
         catch (error) {
+            this.closeImap();
             const classified = classifyError(error, 'SMTP_SEND');
             return {
                 status: submissionStarted &&
@@ -777,6 +860,7 @@ export class ImapSmtpAdapter {
             return { ok: true, value: { providerMessageId: id } };
         }
         catch (error) {
+            this.closeImap();
             if (error.imapCopyRejected &&
                 !state?.destination)
                 remoteStarted = false;
@@ -880,21 +964,27 @@ export class ImapSmtpAdapter {
             return { ok: true, value: undefined };
         }
         catch (error) {
+            this.closeImap();
             return { ok: false, error: classifyError(error, 'IMAP_DELETE_MESSAGE') };
         }
     }
     async close() {
+        this.stopCommandObservation?.();
+        this.stopCommandObservation = undefined;
         this.smtpTransport.close();
-        if (!this.imapClient)
+        const client = this.imapClient;
+        this.imapClient = undefined;
+        if (!client)
             return;
-        try {
-            await this.imapClient.logout();
-        }
-        catch {
-            this.imapClient.close();
-        }
-        finally {
-            this.imapClient = undefined;
+        if (this.pool)
+            await this.pool.release(client);
+        else {
+            try {
+                await client.logout();
+            }
+            catch {
+                client.close();
+            }
         }
     }
     async updateFlags(providerMessageId, flag, enabled, signal) {
@@ -908,22 +998,53 @@ export class ImapSmtpAdapter {
             return { ok: true, value: undefined };
         }
         catch (error) {
+            this.closeImap();
             return { ok: false, error: classifyError(error, 'IMAP_UPDATE_FLAGS') };
         }
     }
     closeImap() {
+        this.stopCommandObservation?.();
+        this.stopCommandObservation = undefined;
         const client = this.imapClient;
         this.imapClient = undefined;
-        client?.close();
+        if (client) {
+            if (this.pool)
+                this.pool.destroy(client);
+            else
+                client.close();
+        }
     }
     async imap() {
         if (this.imapClient)
             return this.imapClient;
-        const client = this.protocol.imap(this.config.imap, this.credential);
-        await client.connect();
+        const client = this.pool
+            ? await this.pool.acquire(this.account.id, this.config.imap, this.credential)
+            : this.protocol.imap(this.config.imap, this.credential);
+        if (!this.pool) {
+            try {
+                await client.connect();
+            }
+            catch (error) {
+                this.closeImap();
+                client.close();
+                throw error;
+            }
+        }
         this.imapClient = client;
+        if (this.syncRun) {
+            this.stopCommandObservation = observeImapCommands(client, this.commandMetrics.record);
+            this.commandMetrics.instrumented ||=
+                this.stopCommandObservation !== undefined;
+        }
         return client;
     }
+    syncDiagnostics() {
+        return {
+            imapCommands: this.commandMetrics.snapshot(),
+            yielded: this.yieldReasons.size > 0,
+            yieldReasons: [...this.yieldReasons],
+        };
+    }
     async listMailboxes() {
         const client = await this.imap();
         const mailboxes = await client.list({
@@ -991,6 +1112,7 @@ export class ImapSmtpAdapter {
             return selectedState;
         }
         catch (error) {
+            this.closeImap();
             if (error?.code !== 'IMAP_INVALID_UIDVALIDITY') {
                 throw error;
             }
@@ -1023,7 +1145,7 @@ export class ImapSmtpAdapter {
             kind: 'folder',
         };
     }
-    async fetchMessages(folder, uidValidity, uids, signal) {
+    async fetchMessages(folder, uidValidity, uids, signal, shouldYield) {
         if (uids.length === 0)
             return [];
         const observed = this.locations
@@ -1039,6 +1161,10 @@ export class ImapSmtpAdapter {
             metadata.push(message);
         }
         const messages = [];
+        if (shouldYield && Array.isArray(uids)) {
+            const order = new Map(uids.map((uid, index) => [uid, index]));
+            metadata.sort((a, b) => order.get(a.uid) - order.get(b.uid));
+        }
         for (const message of metadata) {
             throwIfAborted(signal);
             const normalized = await this.normalizeMessage(folder, uidValidity, message, signal);
@@ -1053,9 +1179,54 @@ export class ImapSmtpAdapter {
                 structure: message.bodyStructure,
                 size: message.size,
             });
+            if (await shouldYield?.())
+                break;
         }
         return messages;
     }
+    /** Check only between completed commands; issuing commands inside FETCH deadlocks. */
+    newMailChecker(folders, signal) {
+        let lastUidCheck = Date.now();
+        let lastPushCheck = Date.now();
+        const check = async () => {
+            throwIfAborted(signal);
+            const now = Date.now();
+            if (this.locations && this.syncRun && now - lastPushCheck >= 1_000) {
+                lastPushCheck = now;
+                const requestedAt = await this.locations.getPushRequestedAt();
+                if (requestedAt &&
+                    Date.parse(requestedAt) > Date.parse(this.syncRun.updatedAt)) {
+                    this.yieldReasons.add('push');
+                    return true;
+                }
+            }
+            if (now - lastUidCheck < 4_000)
+                return false;
+            lastUidCheck = now;
+            const client = await this.imap();
+            if (client.capabilities.has('LIST-STATUS')) {
+                const changed = this.selectableMailboxes(await this.listMailboxes()).some((box) => box.status?.uidNext !== undefined &&
+                    box.status.uidNext !== folders[box.path]?.uidNext);
+                if (changed)
+                    this.yieldReasons.add('uidnext');
+                return changed;
+            }
+            if (!folders.INBOX)
+                return false;
+            const status = await client.status('INBOX', { uidNext: true });
+            if (!status)
+                throw new Error('The IMAP UIDNEXT check was incomplete.');
+            const changed = status.uidNext !== folders.INBOX.uidNext;
+            if (changed)
+                this.yieldReasons.add('uidnext');
+            return changed;
+        };
+        return Object.assign(check, {
+            uidChecked: () => {
+                lastUidCheck = Date.now();
+            },
+        });
+    }
     beginStep() {
         this.copies = [];
         this.changes = [];
@@ -1204,6 +1375,7 @@ export class ImapSmtpAdapter {
                         : undefined;
             }
             catch (error) {
+                this.closeImap();
                 const classified = classifyError(error, 'IMAP_READ_SOURCE');
                 if (classified.retryable || classified.category === 'authentication')
                     throw error;
@@ -1224,29 +1396,12 @@ export class ImapSmtpAdapter {
             if (String(mailbox.uidValidity) !== location.uidValidity)
                 return undefined;
             const message = await client.fetchOne(location.uid, { ...query, uid: true, envelope: true }, { uid: true });
-            if (!message || message.uid !== location.uid)
-                return undefined;
-            const source = location.source ?? 'locator';
-            if (source === 'objectid' || source === 'x-gm-msgid') {
-                const currentSource = this.nativeSource();
-                const nativeId = validNativeId(currentSource, message.emailId);
-                // A UID cannot identify another message within the same UIDVALIDITY.
-                // Missing native metadata preserves this previously verified tuple.
-                return !nativeId ||
-                    (currentSource === source && nativeId === location.providerMessageId)
-                    ? message
-                    : undefined;
-            }
-            if (source === 'rfc-message-id')
-                return message.envelope?.messageId === location.providerMessageId
-                    ? message
-                    : undefined;
-            return source === 'locator' &&
-                encodeCopyLocator(location, location.id) === location.providerMessageId
+            return message && copyMatches(location, message, this.nativeSource())
                 ? message
                 : undefined;
         }
         catch (error) {
+            this.closeImap();
             const classified = classifyError(error, 'IMAP_OPEN_COPY');
             if (classified.retryable || classified.category === 'authentication')
                 throw error;
@@ -1283,6 +1438,7 @@ export class ImapSmtpAdapter {
             return { ok: true, value: undefined };
         }
         catch (error) {
+            this.closeImap();
             return { ok: false, error: classifyError(error, 'IMAP_UPDATE_FLAGS') };
         }
     }
@@ -1290,15 +1446,40 @@ export class ImapSmtpAdapter {
      * One complete pass over every selectable folder. It reconciles stored
      * copies, and two matching passes in a row authorize orphan cleanup.
      */
-    async verifyFolders(mailboxes, folders, previous, signal) {
+    async verifyFolders(mailboxes, folders, previous, signal, receivedAfter) {
         const attempts = (previous?.attempts ?? 0) + 1;
         const snapshot = {};
         const present = new Map();
-        const excluded = new Set(previous?.excluded ?? []);
+        const observedCopies = new Map();
+        const excluded = {
+            ...previous?.excluded,
+        };
         const uncertainGenerations = new Set();
         let stable = true;
+        const checkNewMail = this.newMailChecker(folders, signal);
+        const interrupted = () => {
+            if (!this.yieldReasons.size)
+                this.yieldReasons.add('uidnext');
+            // Partial observations must never invalidate copies or publish flags.
+            this.copies = [];
+            this.changes = [];
+            if (attempts >= MAX_IMAP_VERIFY_ATTEMPTS)
+                this.verificationIncomplete = true;
+            return {
+                done: attempts >= MAX_IMAP_VERIFY_ATTEMPTS,
+                verify: {
+                    attempts,
+                    folders: previous?.folders,
+                    excluded,
+                },
+            };
+        };
+        const changed = (boxes) => boxes.some((box) => box.status?.uidNext !== undefined &&
+            box.status.uidNext !== folders[box.path]?.uidNext);
         const known = await this.locations.listKnownLocations();
         for (const mailbox of mailboxes) {
+            if (await checkNewMail())
+                return interrupted();
             throwIfAborted(signal);
             const client = await this.imap();
             const selected = await client.mailboxOpen(mailbox.path, {
@@ -1310,18 +1491,18 @@ export class ImapSmtpAdapter {
                 return 'generation-changed';
             // New mail arrived after this page: import it before verifying again.
             if (state.uidNext !== cursor.uidNext)
-                stable = false;
+                return interrupted();
             let uids;
             try {
                 uids = await client.search({ all: true }, { uid: true });
             }
             catch (error) {
+                this.closeImap();
                 if (!isResponseLimitError(error))
                     throw error;
                 // The UID list exceeds the protocol budget: keep every copy and skip cleanup.
-                this.closeImap();
                 this.verificationIncomplete = true;
-                return { done: true, verify: { attempts, excluded: [...excluded] } };
+                return { done: true, verify: { attempts, excluded } };
             }
             if (!Array.isArray(uids))
                 // Retry the step: an incomplete SEARCH proves nothing about missing copies.
@@ -1333,22 +1514,115 @@ export class ImapSmtpAdapter {
             if (Buffer.byteLength(JSON.stringify(snapshot)) >
                 MAX_IMAP_VERIFY_STATE_BYTES) {
                 this.verificationIncomplete = true;
-                return { done: true, verify: { attempts, excluded: [...excluded] } };
+                return { done: true, verify: { attempts, excluded } };
             }
             const bound = new Set(known
                 .filter((location) => location.folder === mailbox.path &&
                 location.uidValidity === state.uidValidity &&
                 location.status === 'valid')
                 .map((location) => location.uid));
-            const unbound = uids.filter((uid) => !bound.has(uid) &&
-                !excluded.has(tupleKey({
-                    folder: mailbox.path,
+            const candidates = known.filter((location) => location.folder === mailbox.path &&
+                location.uidValidity === state.uidValidity &&
+                location.status === 'valid' &&
+                location.source &&
+                present.get(mailbox.path).has(location.uid));
+            for (let offset = 0; offset < candidates.length; offset += 1000) {
+                if (await checkNewMail())
+                    return interrupted();
+                throwIfAborted(signal);
+                const batch = candidates.slice(offset, offset + 1000);
+                const messages = await client.fetchAll(compressUids(batch.map((location) => location.uid)), {
+                    uid: true,
+                    flags: true,
+                    ...(batch.some((location) => location.source === 'objectid' ||
+                        location.source === 'x-gm-msgid')
+                        ? { emailId: true }
+                        : {}),
+                    ...(batch.some((location) => location.source === 'rfc-message-id')
+                        ? { envelope: true }
+                        : {}),
+                }, { uid: true });
+                const byUid = new Map(messages.map((message) => [message.uid, message]));
+                for (const location of batch) {
+                    throwIfAborted(signal);
+                    const message = byUid.get(location.uid);
+                    if (message?.flags &&
+                        copyMatches(location, message, this.nativeSource()))
+                        observedCopies.set(location.id, message);
+                    else
+                        stable = false;
+                }
+            }
+            const excludedUids = new Set(matchingUids(uids, excluded[mailbox.path]?.uidValidity === state.uidValidity
+                ? excluded[mailbox.path].uids
+                : ''));
+            const rememberExcluded = () => {
+                if (!excludedUids.size) {
+                    delete excluded[mailbox.path];
+                    return;
+                }
+                excluded[mailbox.path] = {
                     uidValidity: state.uidValidity,
-                    uid,
-                })));
-            if (unbound.length) {
-                // Reconcile copies below the scan high-water mark, including old-date moves.
-                const fetched = await this.fetchMessages(mailbox.path, state.uidValidity, unbound.slice(0, this.syncRun?.policy.batchSize ?? 100), signal);
+                    uids: compressUids(excludedUids),
+                };
+            };
+            const unbound = uids.filter((uid) => !bound.has(uid) && !excludedUids.has(uid));
+            const knownUids = new Set(known
+                .filter((location) => location.folder === mailbox.path &&
+                location.uidValidity === state.uidValidity)
+                .map((location) => location.uid));
+            const toFetch = [];
+            const bodyLimit = this.syncRun?.policy.batchSize ?? 100;
+            // Screen large historical mailboxes without downloading old bodies. A copy
+            // of an existing identity still needs full evidence, even with an old date.
+            for (let offset = 0; offset < unbound.length; offset += 1000) {
+                if (await checkNewMail()) {
+                    rememberExcluded();
+                    return interrupted();
+                }
+                const batch = unbound.slice(offset, offset + 1000);
+                const metadata = await client.fetchAll(compressUids(batch), {
+                    uid: true,
+                    internalDate: true,
+                    emailId: true,
+                    headers: ['message-id'],
+                }, { uid: true });
+                const sources = await this.locations.findMessageSources(metadata.flatMap((message) => {
+                    const id = validNativeId(this.nativeSource(), message.emailId) ??
+                        validRfcMessageId(messageIdHeader(message.headers));
+                    return id ? [id] : [];
+                }));
+                for (const message of metadata) {
+                    const nativeId = validNativeId(this.nativeSource(), message.emailId);
+                    const id = nativeId ?? validRfcMessageId(messageIdHeader(message.headers));
+                    const identityReadable = nativeId ||
+                        (message.headers &&
+                            message.headers.byteLength <= MAX_HEADER_BYTES &&
+                            (message.headers.equals(Buffer.from('\r\n')) ||
+                                message.headers.subarray(-4).equals(Buffer.from('\r\n\r\n'))));
+                    const receivedAt = toIsoDate(message.internalDate);
+                    const observed = knownUids.has(message.uid);
+                    if (receivedAfter &&
+                        identityReadable &&
+                        receivedAt &&
+                        Date.parse(receivedAt) < Date.parse(receivedAfter) &&
+                        !observed &&
+                        (!id || !sources.has(id)))
+                        excludedUids.add(message.uid);
+                    else
+                        toFetch.push(message.uid);
+                }
+                rememberExcluded();
+                // Bound the full-body work; remaining UIDs stay unbound for the next pass.
+                if (toFetch.length >= bodyLimit)
+                    break;
+            }
+            rememberExcluded();
+            if (toFetch.length) {
+                if (await checkNewMail())
+                    return interrupted();
+                let yielded = false;
+                const fetched = await this.fetchMessages(mailbox.path, state.uidValidity, toFetch.slice(0, bodyLimit), signal, async () => (yielded = await checkNewMail()));
                 const sources = await this.locations.findMessageSources(fetched.flatMap((copy) => {
                     const id = validNativeId(this.nativeSource(), copy.emailId) ??
                         copy.rfcMessageId;
@@ -1357,25 +1631,37 @@ export class ImapSmtpAdapter {
                 for (const copy of fetched) {
                     const id = validNativeId(this.nativeSource(), copy.emailId) ??
                         copy.rfcMessageId;
-                    const beforeImportDate = this.syncRun?.policy.receivedAfter &&
+                    if (receivedAfter &&
                         copy.message.receivedAt &&
-                        Date.parse(copy.message.receivedAt) <
-                            Date.parse(this.syncRun.policy.receivedAfter);
-                    if (beforeImportDate && !copy.observed && (!id || !sources.has(id)))
-                        excluded.add(tupleKey(copy.tuple));
+                        Date.parse(copy.message.receivedAt) < Date.parse(receivedAfter) &&
+                        !copy.observed &&
+                        (!id || !sources.has(id)))
+                        excludedUids.add(copy.tuple.uid);
                     else
                         this.verificationCopies.push(copy);
                 }
-                stable = false;
+                rememberExcluded();
+                if (yielded)
+                    return interrupted();
             }
+            if (unbound.some((uid) => !excludedUids.has(uid)))
+                stable = false;
         }
         const after = this.selectableMailboxes(await this.listMailboxes());
+        checkNewMail.uidChecked();
+        if (changed(after))
+            return interrupted();
         if (after.length !== mailboxes.length ||
             after.some((box) => !snapshot[box.path]))
             stable = false;
         for (const mailbox of after) {
+            if (await checkNewMail())
+                return interrupted();
+            throwIfAborted(signal);
             const selected = await (await this.imap()).mailboxOpen(mailbox.path, { readOnly: true });
             const state = await this.selectedFolderState(selected);
+            if (state.uidNext !== folders[mailbox.path]?.uidNext)
+                return interrupted();
             const rechecked = await (await this.imap()).search({ all: true }, { uid: true });
             if (!Array.isArray(rechecked))
                 throw new Error('The final IMAP SEARCH was incomplete.');
@@ -1388,11 +1674,14 @@ export class ImapSmtpAdapter {
                 uncertainGenerations.add(mailbox.path);
         }
         const finalRange = this.selectableMailboxes(await this.listMailboxes());
+        if (changed(finalRange))
+            return interrupted();
         const finalPaths = new Set(finalRange.map((mailbox) => mailbox.path));
         if (finalPaths.size !== mailboxes.length ||
             mailboxes.some((mailbox) => !finalPaths.has(mailbox.path)))
             stable = false;
         for (const mailbox of finalRange) {
+            throwIfAborted(signal);
             const generation = mailbox.status?.uidValidity;
             const uidNext = mailbox.status?.uidNext;
             if (generation !== undefined &&
@@ -1404,6 +1693,7 @@ export class ImapSmtpAdapter {
                 stable = false;
         }
         for (const location of known) {
+            throwIfAborted(signal);
             if (this.verificationCopies.some((copy) => tupleKey(copy.tuple) === tupleKey(location)))
                 continue;
             const uids = present.get(location.folder);
@@ -1421,9 +1711,11 @@ export class ImapSmtpAdapter {
                 location.source &&
                 generation === location.uidValidity &&
                 uids?.has(location.uid)) {
-                const observed = await this.openCopy(location, { flags: true });
-                if (!observed?.flags)
-                    throw new Error('The IMAP copy identity or flags changed during verification.');
+                const observed = observedCopies.get(location.id);
+                if (!observed?.flags) {
+                    stable = false;
+                    continue;
+                }
                 this.copies.push({
                     flagsOnly: true,
                     locationId: location.id,
@@ -1453,29 +1745,122 @@ export class ImapSmtpAdapter {
             if (next.status !== 'valid' || next.status !== location.status)
                 this.changes.push({ id: location.id, observed: location, ...next });
         }
-        if (Buffer.byteLength(JSON.stringify({ snapshot, excluded: [...excluded] })) >
+        if (Buffer.byteLength(JSON.stringify({ snapshot, excluded })) >
             MAX_IMAP_VERIFY_STATE_BYTES) {
             this.verificationIncomplete = true;
-            return { done: true, verify: { attempts } };
+            return { done: true, verify: { attempts, excluded } };
         }
         if (!stable) {
             if (attempts >= MAX_IMAP_VERIFY_ATTEMPTS) {
                 this.verificationIncomplete = true;
-                return { done: true, verify: { attempts, excluded: [...excluded] } };
+                return { done: true, verify: { attempts, excluded } };
             }
-            return { done: false, verify: { attempts, excluded: [...excluded] } };
+            return { done: false, verify: { attempts, excluded } };
         }
-        if (previous?.folders && sameSnapshot(previous.folders, snapshot)) {
+        if (previous?.folders &&
+            sameSnapshot(previous.folders, snapshot) &&
+            // A newly imported then expunged copy needs its own complete missing pass.
+            !known.some((location) => location.uidValidity ===
+                previous.folders?.[location.folder]?.uidValidity &&
+                location.uid >= previous.folders[location.folder].uidNext &&
+                !present.get(location.folder)?.has(location.uid))) {
             this.cleanup = true;
-            return { done: true };
+            return { done: true, verify: { excluded } };
         }
         if (attempts >= MAX_IMAP_VERIFY_ATTEMPTS) {
             this.verificationIncomplete = true;
-            return { done: true, verify: { attempts, excluded: [...excluded] } };
+            return { done: true, verify: { attempts, excluded } };
         }
         return {
             done: false,
-            verify: { attempts, folders: snapshot, excluded: [...excluded] },
+            verify: { attempts, folders: snapshot, excluded },
+        };
+    }
+    enqueueExcludedRechecks(verify, copies) {
+        const folders = Object.keys(verify.excluded ?? {}).filter((path) => verify.excluded[path].uids);
+        if (!folders.length)
+            return verify;
+        const pending = [...(verify.rechecks ?? [])];
+        const key = (identity) => identity.emailId !== undefined
+            ? 'native:' + identity.emailId
+            : 'rfc:' + identity.rfcMessageId;
+        const queued = new Set(pending.flatMap((batch) => batch.identities.map(key)));
+        const identities = new Map();
+        for (const copy of copies) {
+            const source = this.nativeSource();
+            const nativeId = validNativeId(source, copy.emailId ??
+                (copy.observed?.source === source
+                    ? copy.observed?.providerMessageId
+                    : undefined));
+            const identity = nativeId
+                ? { emailId: nativeId }
+                : copy.rfcMessageId
+                    ? { rfcMessageId: copy.rfcMessageId }
+                    : undefined;
+            if (identity && !queued.has(key(identity)))
+                identities.set(key(identity), identity);
+        }
+        if (pending.reduce((total, batch) => total + batch.identities.length, 0) +
+            identities.size >
+            MAX_IMAP_RECHECK_IDENTITIES) {
+            // Bounded cursor fallback: re-screen later, after the arrival is committed.
+            return { attempts: 0 };
+        }
+        const added = [...identities.values()];
+        for (let offset = 0; offset < added.length; offset += IMAP_RECHECK_BATCH_SIZE)
+            pending.push({
+                identities: added.slice(offset, offset + IMAP_RECHECK_BATCH_SIZE),
+                folders,
+            });
+        return { ...verify, rechecks: pending.length ? pending : undefined };
+    }
+    async advanceExcludedRecheck(verify, folders, signal) {
+        const excluded = { ...verify.excluded };
+        const pending = [...(verify.rechecks ?? [])];
+        while (pending.length) {
+            throwIfAborted(signal);
+            const batch = pending[0];
+            const [path, ...remaining] = batch.folders;
+            const advance = () => {
+                if (remaining.length)
+                    pending[0] = { ...batch, folders: remaining };
+                else
+                    pending.shift();
+            };
+            const range = excluded[path];
+            if (!range?.uids || folders[path]?.uidValidity !== range.uidValidity) {
+                delete excluded[path];
+                advance();
+                continue;
+            }
+            const client = await this.imap();
+            const selected = await client.mailboxOpen(path, { readOnly: true });
+            if (String(selected.uidValidity) !== range.uidValidity) {
+                delete excluded[path];
+                advance();
+                continue;
+            }
+            const identities = batch.identities.map((identity) => identity.emailId !== undefined
+                ? { emailId: identity.emailId }
+                : { header: { 'message-id': identity.rfcMessageId } });
+            const found = await client.search({
+                uid: range.uids,
+                ...(identities.length === 1 ? identities[0] : { or: identities }),
+            }, { uid: true });
+            if (!Array.isArray(found))
+                throw new Error('The excluded-copy identity SEARCH was incomplete.');
+            const uids = subtractUids(range.uids, found);
+            if (uids)
+                excluded[path] = { ...range, uids };
+            else
+                delete excluded[path];
+            advance();
+            break; // One folder per committed step; arrivals take priority on the next step.
+        }
+        return {
+            ...verify,
+            excluded,
+            rechecks: pending.length ? pending : undefined,
         };
     }
     async normalizeMessage(folder, uidValidity, message, signal, maxBodyBytes = MAX_IMAP_BODY_BYTES) {
@@ -1524,7 +1909,7 @@ export class ImapSmtpAdapter {
             ...body,
             size: message.size,
             // Preserve the header value; mailparser can wrap malformed bare IDs in brackets.
-            internetMessageId: /^message-id:[\t ]*([^\r\n]*(?:\r\n[\t ][^\r\n]*)*)\r\n/im.exec(header.toString('utf8'))?.[1],
+            internetMessageId: messageIdHeader(header),
             providerFolderIds: [folder],
             from,
             to,
@@ -1572,8 +1957,23 @@ function sameSnapshot(left, right) {
     const paths = Object.keys(right);
     return (Object.keys(left).length === paths.length &&
         paths.every((path) => left[path]?.uidValidity === right[path].uidValidity &&
-            left[path].uidNext === right[path].uidNext &&
-            left[path].uids === right[path].uids));
+            left[path].uidNext <= right[path].uidNext &&
+            left[path].uids === uidsBelow(right[path].uids, left[path].uidNext)));
+}
+/** Clip compressed ranges without expanding potentially very large UID sets. */
+function uidsBelow(ranges, uidNext) {
+    return ranges
+        .split(',')
+        .flatMap((range) => {
+        if (!range)
+            return [];
+        const [start, end = start] = range.split(':').map(Number);
+        if (start >= uidNext)
+            return [];
+        const last = Math.min(end, uidNext - 1);
+        return [start === last ? String(start) : `${start}:${last}`];
+    })
+        .join(',');
 }
 function isResponseLimitError(error) {
     return ['LiteralTooLarge', 'ResponseTooLarge', 'LineTooLarge'].includes(error?.code ?? '');
@@ -1641,4 +2041,25 @@ async function transferUid(client, uid, folder, native) {
     parse(result.response);
     return invalid ? undefined : mapping;
 }
+/** Compare identity metadata without reopening the selected mailbox. */
+function copyMatches(location, message, currentSource) {
+    if (message.uid !== location.uid)
+        return false;
+    const source = location.source ?? 'locator';
+    if (source === 'objectid' || source === 'x-gm-msgid') {
+        const nativeId = validNativeId(currentSource, message.emailId);
+        // Missing native metadata preserves this previously verified tuple.
+        return (!nativeId ||
+            (currentSource === source && nativeId === location.providerMessageId));
+    }
+    if (source === 'rfc-message-id')
+        return message.envelope?.messageId === location.providerMessageId;
+    return (source === 'locator' &&
+        encodeCopyLocator(location, location.id) === location.providerMessageId);
+}
+/** Preserve the raw header value, including malformed IDs, on both FETCH paths. */
+function messageIdHeader(header) {
+    return (header &&
+        /^message-id:[\t ]*([^\r\n]*(?:\r\n[\t ][^\r\n]*)*)\r\n/im.exec(header.toString('utf8'))?.[1]);
+}
 //# sourceMappingURL=adapter.js.map