@nocobase/app-plugin-mail 1.0.0-beta.13 → 1.0.0-beta.14

package/dist/server/adapters/imap-smtp/adapter.js

@@ -2,10 +2,10 @@ import MailComposer from 'nodemailer/lib/mail-composer/index.js';
 import { randomUUID } from 'node:crypto';
 import { simpleParser } from 'mailparser';
 import { Readable, addAbortSignal } from 'node:stream';
-import { attachmentParts, bodyParts, readBody, metadataQuery, attachmentMetadata, } from './content.js';
-import { IMAP_SMTP_CAPABILITIES, MAX_IMAP_BODY_BYTES, MAX_IMAP_DETAIL_BODY_BYTES, } from './constants.js';
+import { attachmentParts, bodyParts, readBody, readRawSource, rawComparable, metadataQuery, attachmentMetadata, MAX_HEADER_BYTES, } from './content.js';
+import { IMAP_SMTP_CAPABILITIES, MAX_IMAP_BODY_BYTES, MAX_IMAP_DETAIL_BODY_BYTES, MAX_IMAP_PROVIDER_ID_LENGTH, MAX_IMAP_RAW_COMPARE_BUDGET, MAX_IMAP_RAW_COMPARE_BYTES, MAX_IMAP_VERIFY_ATTEMPTS, MAX_IMAP_VERIFY_STATE_BYTES, } from './constants.js';
 import { createImapClient, createSmtpTransport } from './connection.js';
-import { encodeHistoryCursor, encodeMessageLocator, folderCursor, parseHistoryCursor, parseMessageLocator, parseSyncCursor, rangeFor, requireAttachmentLocator, requireMessageLocator, syncCursor, } from './locators.js';
+import { compressUids, encodeCopyLocator, encodeHistoryCursor, encodeMessageLocator, folderCursor, parseHistoryCursor, parseSyncCursor, rangeFor, requireAttachmentLocator, requireMessageLocator, syncCursor, } from './locators.js';
 import { classifyFolderType, folderTypeFromPath } from './folders.js';
 function classifyConfiguredFolder(mailbox, sentFolder) {
     if (sentFolder && mailbox.path === sentFolder)
@@ -15,18 +15,104 @@ function classifyConfiguredFolder(mailbox, sentFolder) {
 import { addresses, envelopeAddress, envelopeAddresses, preview, references, toHeader, toIsoDate, } from './normalize.js';
 import { classifyError, failure, throwIfAborted } from './errors.js';
 export class ImapSmtpAdapter {
+    context;
     config;
+    account;
     credential;
+    locations;
+    protocol;
+    operation;
     identity;
     capabilities = IMAP_SMTP_CAPABILITIES;
     folderTypes = new Map();
     imapClient;
     smtpTransport;
-    constructor(_context, config, account, credential) {
+    // Evidence and byte budget of the current step only; never reused across steps.
+    copies = [];
+    changes = [];
+    cleanup = false;
+    verificationIncomplete = false;
+    verificationCopies = [];
+    syncRun;
+    rawBudget = MAX_IMAP_RAW_COMPARE_BUDGET;
+    rawCache = new Map();
+    constructor(context, config, account, credential, locations, run, protocol = Object.freeze({
+        imap: createImapClient,
+        smtp: createSmtpTransport,
+        expungeUid,
+    }), operation) {
+        this.context = context;
         this.config = config;
+        this.account = account;
         this.credential = credential;
-        this.identity = account.provider;
-        this.smtpTransport = createSmtpTransport(config.smtp, credential);
+        this.locations = locations;
+        this.protocol = protocol;
+        this.operation = operation;
+        if (locations && locations.accountId !== account.id)
+            throw new Error('The IMAP persistence belongs to another account.');
+        if (operation && operation.accountId !== account.id)
+            throw new Error('The IMAP operation belongs to another account.');
+        this.operation = operation && Object.freeze({ ...operation });
+        this.config = Object.freeze({
+            ...config,
+            imap: Object.freeze({ ...config.imap }),
+            smtp: Object.freeze({ ...config.smtp }),
+        });
+        this.account = Object.freeze({
+            ...account,
+            provider: Object.freeze({ ...account.provider }),
+        });
+        this.credential = Object.freeze({ ...credential });
+        this.syncRun =
+            run &&
+                Object.freeze({ ...run, policy: Object.freeze({ ...run.policy }) });
+        this.protocol = Object.freeze({ ...protocol });
+        this.identity = this.account.provider;
+        this.smtpTransport = this.protocol.smtp(this.config.smtp, this.credential);
+    }
+    forSync(run) {
+        if (!this.locations)
+            throw new Error('IMAP location persistence is not configured.');
+        if (this.operation)
+            throw new Error('An IMAP message operation cannot be reused for synchronization.');
+        if (run.accountId !== this.account.id)
+            throw new Error('The IMAP sync account changed.');
+        return new ImapSmtpAdapter(this.context, this.config, this.account, this.credential, this.locations, run, this.protocol);
+    }
+    async forMessage(messageId, providerMessageId, operation) {
+        if (!this.locations)
+            throw new Error('IMAP location persistence is not configured.');
+        const bound = await this.locations.bindMessage(messageId, providerMessageId, operation);
+        return new ImapSmtpAdapter(this.context, this.config, this.account, this.credential, bound.persistence, undefined, this.protocol, bound.snapshot);
+    }
+    /** True when this instance records message copies in the account's location table. */
+    get persistsLocations() {
+        return this.locations !== undefined;
+    }
+    /** Commit a sync step together with the location evidence this instance gathered for it. */
+    async commitSyncStep(input) {
+        if (!this.locations)
+            throw new Error('IMAP location persistence is not configured.');
+        if (!this.syncRun)
+            throw new Error('An IMAP sync commit requires its immutable run snapshot.');
+        const ids = new Set(input.messages.map((message) => message.providerMessageId));
+        if (this.syncRun &&
+            (input.run.id !== this.syncRun.id ||
+                input.run.accountId !== this.syncRun.accountId ||
+                input.run.phase !== this.syncRun.phase ||
+                input.run.revision !== this.syncRun.revision ||
+                input.run.leaseToken !== this.syncRun.leaseToken))
+            throw new Error('The IMAP sync operation snapshot changed.');
+        const incomplete = this.verificationIncomplete;
+        const evidence = {
+            copies: this.copies.filter((copy) => copy.flagsOnly || ids.has(copy.providerMessageId)),
+            changes: this.changes,
+            cleanup: this.cleanup,
+        };
+        this.beginStep();
+        return this.locations.commitSyncStep(incomplete
+            ? { ...input, status: 'partial', createNextTask: false }
+            : input, evidence);
     }
     async listFolders(input) {
         try {
@@ -63,6 +149,7 @@ export class ImapSmtpAdapter {
         }
     }
     async listMessages(input) {
+        this.beginStep();
         try {
             throwIfAborted(input.signal);
             const folders = input.providerFolderIds?.length
@@ -150,13 +237,14 @@ export class ImapSmtpAdapter {
                         upperUid = start - 1;
                     }
                 }
-                for (const message of await this.fetchMessages(folder, String(mailbox.uidValidity), selected, input.signal)) {
+                for (const copy of await this.fetchMessages(folder, String(mailbox.uidValidity), selected, input.signal)) {
+                    const { receivedAt } = copy.message;
                     if (input.receivedAfter &&
-                        message.receivedAt &&
-                        Date.parse(message.receivedAt) < Date.parse(input.receivedAfter)) {
+                        receivedAt &&
+                        Date.parse(receivedAt) < Date.parse(input.receivedAfter)) {
                         continue;
                     }
-                    messages.push(message);
+                    messages.push(copy);
                 }
                 if (upperUid !== undefined && upperUid < 1) {
                     folderIndex += 1;
@@ -169,7 +257,7 @@ export class ImapSmtpAdapter {
             return {
                 ok: true,
                 value: {
-                    messages,
+                    messages: await this.assignIdentities(messages, input.signal),
                     nextCursor,
                     syncCursor: input.cursor ? undefined : input.baselineCursor,
                 },
@@ -180,6 +268,7 @@ export class ImapSmtpAdapter {
         }
     }
     async listChanges(input) {
+        this.beginStep();
         try {
             throwIfAborted(input.signal);
             let previous;
@@ -200,78 +289,40 @@ export class ImapSmtpAdapter {
             const mailboxes = this.selectableMailboxes(await this.listMailboxes());
             const nextFolders = {};
             const messages = [];
-            const deletedProviderMessageIds = new Set();
-            const knownByFolder = new Map();
-            for (const providerMessageId of input.knownProviderMessageIds ?? []) {
-                const locator = parseMessageLocator(providerMessageId);
-                if (!locator)
-                    continue;
-                const known = knownByFolder.get(locator.folder) ?? new Set();
-                known.add(providerMessageId);
-                knownByFolder.set(locator.folder, known);
-            }
             let hasMore = false;
             const limit = Math.max(1, input.limit);
             for (const mailbox of mailboxes) {
                 throwIfAborted(input.signal);
                 const current = await this.folderState(mailbox.path, mailbox);
                 const old = previous.folders[mailbox.path];
-                if (old && old.uidValidity !== current.uidValidity) {
-                    return {
-                        ok: false,
-                        error: {
-                            code: 'IMAP_SYNC_CURSOR_INVALID',
-                            message: `The IMAP UIDVALIDITY changed for ${mailbox.path}.`,
-                            category: 'provider',
-                            retryable: false,
-                        },
-                    };
-                }
-                const start = old?.uidNext ?? 1;
+                const sameGeneration = old?.uidValidity === current.uidValidity;
+                const start = sameGeneration ? old.uidNext : 1;
                 const canFetch = messages.length < limit;
                 const end = canFetch
                     ? Math.min(current.uidNext - 1, start + limit - messages.length - 1)
                     : start - 1;
-                let nextUid = old?.uidNext ?? 1;
-                const knownIds = knownByFolder.get(mailbox.path);
-                if (start <= end || knownIds?.size) {
+                let nextUid = sameGeneration ? old.uidNext : 1;
+                if (start <= end) {
                     const client = await this.imap();
                     const selected = await client.mailboxOpen(mailbox.path, {
                         readOnly: true,
                     });
                     if (String(selected.uidValidity) !== current.uidValidity)
                         return failure('IMAP_SYNC_CURSOR_INVALID', 'Mailbox generation changed before fetching.', 'provider', false);
-                    if (knownIds?.size) {
-                        const result = await client.search({ all: true }, { uid: true });
-                        if (!Array.isArray(result))
-                            return failure('IMAP_SEARCH_FAILED', `Could not search ${mailbox.path} while reconciling deleted messages.`, 'provider', true);
-                        const presentIds = new Set(result.map((uid) => encodeMessageLocator({
-                            folder: mailbox.path,
-                            uidValidity: current.uidValidity,
-                            uid,
-                        })));
-                        for (const providerMessageId of knownIds) {
-                            if (!presentIds.has(providerMessageId)) {
-                                deletedProviderMessageIds.add(providerMessageId);
-                            }
-                        }
-                    }
-                    if (start <= end) {
-                        const fetched = await this.fetchMessages(mailbox.path, current.uidValidity, rangeFor(start, end), input.signal);
-                        messages.push(...fetched);
-                        // Complete the bounded UID range before advancing, regardless of response order or holes.
-                        nextUid = end + 1;
-                        if (fetched.length === 0 && nextUid < current.uidNext) {
-                            // Some servers assign very large, sparse UIDs. Jump to the next
-                            // existing message instead of scheduling every empty UID range.
-                            const remaining = await client.search({ uid: rangeFor(nextUid, current.uidNext - 1) }, { uid: true });
-                            const lowerBound = nextUid;
-                            nextUid = current.uidNext;
-                            if (Array.isArray(remaining)) {
-                                for (const uid of remaining) {
-                                    if (uid >= lowerBound && uid < nextUid)
-                                        nextUid = uid;
-                                }
+                    const fetched = await this.fetchMessages(mailbox.path, current.uidValidity, rangeFor(start, end), input.signal);
+                    messages.push(...fetched);
+                    // Complete the bounded UID range before advancing, regardless of response order or holes.
+                    nextUid = end + 1;
+                    if (fetched.length === 0 && nextUid < current.uidNext) {
+                        // Some servers assign very large, sparse UIDs. Jump to the next
+                        // existing message instead of scheduling every empty UID range.
+                        const remaining = await client.search({ uid: rangeFor(nextUid, current.uidNext - 1) }, { uid: true });
+                        const lowerBound = nextUid;
+                        nextUid = current.uidNext;
+                        if (Array.isArray(remaining)) {
+                            for (const uid of remaining) {
+                                if (uid >= lowerBound && uid < nextUid)
+                                    nextUid = uid;
                             }
                         }
                     }
@@ -286,12 +337,36 @@ export class ImapSmtpAdapter {
                     uidNext: nextUid,
                 };
             }
+            let verify;
+            if (this.locations) {
+                if (hasMore || messages.length > 0) {
+                    // New copies are committed first; verification restarts afterwards.
+                    verify = previous.verify && {
+                        attempts: previous.verify.attempts,
+                        excluded: previous.verify.excluded,
+                    };
+                    hasMore = true;
+                }
+                else {
+                    const result = await this.verifyFolders(mailboxes, nextFolders, previous.verify, input.signal);
+                    if (result === 'generation-changed')
+                        return failure('IMAP_SYNC_CURSOR_INVALID', 'Mailbox generation changed during verification.', 'provider', false);
+                    messages.push(...this.verificationCopies);
+                    hasMore = !result.done;
+                    verify = result.verify;
+                }
+            }
             return {
                 ok: true,
                 value: {
-                    messages,
-                    deletedProviderMessageIds: [...deletedProviderMessageIds],
-                    nextCursor: syncCursor({ version: 1, folders: nextFolders }),
+                    messages: await this.assignIdentities(messages, input.signal),
+                    // Copies leave through location evidence; one missing UID never deletes a message.
+                    deletedProviderMessageIds: [],
+                    nextCursor: syncCursor({
+                        version: 1,
+                        folders: nextFolders,
+                        ...(verify ? { verify } : {}),
+                    }),
                     hasMore,
                 },
             };
@@ -303,6 +378,22 @@ export class ImapSmtpAdapter {
     async getMessage(providerMessageId, signal) {
         try {
             throwIfAborted(signal);
+            if (this.locations) {
+                // Try each verified copy; a stale copy is skipped, never reported as deleted.
+                for (const location of await this.locations.listValidLocations(providerMessageId)) {
+                    const message = await this.openCopy(location, metadataQuery());
+                    if (!message)
+                        continue;
+                    return {
+                        ok: true,
+                        value: {
+                            ...(await this.normalizeMessage(location.folder, location.uidValidity, message, signal, MAX_IMAP_DETAIL_BODY_BYTES)),
+                            providerMessageId,
+                        },
+                    };
+                }
+                return failure('IMAP_MESSAGE_NOT_FOUND', 'No verified IMAP copy of this message is available.', 'provider', false);
+            }
             const locator = requireMessageLocator(providerMessageId);
             const mailbox = await (await this.imap()).mailboxOpen(locator.folder, {
                 readOnly: true,
@@ -322,24 +413,72 @@ export class ImapSmtpAdapter {
             return { ok: false, error: classifyError(error, 'IMAP_GET_MESSAGE') };
         }
     }
-    async getAttachment(_providerMessageId, providerAttachmentId, signal) {
+    async getAttachment(providerMessageId, providerAttachmentId, signal) {
+        let release;
+        let streamed = false;
         try {
             throwIfAborted(signal);
             const locator = requireAttachmentLocator(providerAttachmentId);
-            const selected = await (await this.imap()).mailboxOpen(locator.folder, { readOnly: true });
+            let storedCopy;
+            let expected;
+            if (this.locations) {
+                // The part index is only meaningful on the copy it was read from.
+                const [location] = await this.locations.findLocations([locator]);
+                if (location?.status !== 'valid' ||
+                    location.providerMessageId !== providerMessageId)
+                    return failure('IMAP_ATTACHMENT_NOT_FOUND', 'The IMAP copy holding this attachment is no longer available.', 'provider', false);
+                storedCopy = location;
+                expected = await this.locations.findAttachment(location, providerAttachmentId);
+                if (!expected)
+                    return failure('IMAP_ATTACHMENT_NOT_FOUND', 'The stored attachment does not belong to this copy.', 'provider', false);
+            }
+            const lockedClient = await this.imap();
+            const lock = await lockedClient.getMailboxLock(locator.folder, {
+                readOnly: true,
+            });
+            let released = false;
+            release = () => {
+                if (!released) {
+                    released = true;
+                    lock.release();
+                }
+            };
+            const selected = lockedClient.mailbox;
+            if (!selected)
+                throw new Error('IMAP mailbox lock did not select the folder.');
             if (String(selected.uidValidity) !== locator.uidValidity)
                 return failure('IMAP_SYNC_CURSOR_INVALID', 'Mailbox generation changed before downloading.', 'provider', false);
             const client = await this.imap();
-            const message = await client.fetchOne(locator.uid, { uid: true, bodyStructure: true }, { uid: true });
-            const part = message && attachmentParts(message.bodyStructure)[locator.attachment];
+            const message = await client.fetchOne(locator.uid, { uid: true, bodyStructure: true, envelope: true }, { uid: true });
+            if (!message || message.uid !== locator.uid)
+                return failure('IMAP_ATTACHMENT_NOT_FOUND', 'The IMAP attachment copy changed.', 'provider', false);
+            const part = attachmentParts(message.bodyStructure)[locator.attachment];
             if (!part)
                 return failure('IMAP_ATTACHMENT_NOT_FOUND', 'IMAP attachment was not found.', 'provider', false);
+            if (expected) {
+                const actual = attachmentMetadata(locator.folder, locator.uidValidity, locator.uid, part, locator.attachment);
+                if (actual.fileName !== expected.fileName ||
+                    actual.contentType !== expected.contentType ||
+                    actual.size !== expected.size ||
+                    actual.contentId !== expected.contentId ||
+                    actual.inline !== expected.inline ||
+                    ((storedCopy?.source === 'objectid' ||
+                        storedCopy?.source === 'x-gm-msgid') &&
+                        validNativeId(this.nativeSource(), message.emailId) !== undefined &&
+                        (this.nativeSource() !== storedCopy.source ||
+                            message.emailId !== storedCopy.providerMessageId)))
+                    return failure('IMAP_ATTACHMENT_NOT_FOUND', 'The IMAP attachment metadata or identity changed.', 'provider', false);
+            }
             const download = await client.download(locator.uid, part.part ?? '1', {
                 uid: true,
                 chunkSize: 64 * 1024,
             });
             if (!download.content)
                 return failure('IMAP_ATTACHMENT_NOT_FOUND', 'IMAP attachment was not found.', 'provider', false);
+            download.content.once('end', release);
+            download.content.once('error', release);
+            download.content.once('close', release);
+            streamed = true;
             if (signal)
                 addAbortSignal(signal, download.content);
             return {
@@ -363,6 +502,10 @@ export class ImapSmtpAdapter {
         catch (error) {
             return { ok: false, error: classifyError(error, 'IMAP_GET_ATTACHMENT') };
         }
+        finally {
+            if (!streamed)
+                release?.();
+        }
     }
     async sendMessage(input) {
         let submissionStarted = false;
@@ -483,21 +626,80 @@ export class ImapSmtpAdapter {
         }
     }
     async setRead(providerMessageId, read, signal) {
-        return this.updateFlags(providerMessageId, '\\Seen', read, signal);
+        if ((this.locations && !this.operation) ||
+            (this.operation &&
+                (this.operation.operation !== 'flags' ||
+                    this.operation.providerMessageId !== providerMessageId)))
+            return failure('IMAP_OPERATION_MISMATCH', 'The IMAP operation does not match its immutable target.', 'configuration', false);
+        return this.locations
+            ? this.updateCopyFlags(providerMessageId, '\\Seen', read, signal)
+            : this.updateFlags(providerMessageId, '\\Seen', read, signal);
     }
     async setStarred(providerMessageId, starred, signal) {
-        return this.updateFlags(providerMessageId, '\\Flagged', starred, signal);
+        if ((this.locations && !this.operation) ||
+            (this.operation &&
+                (this.operation.operation !== 'flags' ||
+                    this.operation.providerMessageId !== providerMessageId)))
+            return failure('IMAP_OPERATION_MISMATCH', 'The IMAP operation does not match its immutable target.', 'configuration', false);
+        return this.locations
+            ? this.updateCopyFlags(providerMessageId, '\\Flagged', starred, signal)
+            : this.updateFlags(providerMessageId, '\\Flagged', starred, signal);
     }
     async deleteMessage(providerMessageId, permanently, signal) {
+        if ((this.locations && !this.operation) ||
+            (this.operation &&
+                (this.operation.operation !== 'delete' ||
+                    this.operation.providerMessageId !== providerMessageId)))
+            return failure('IMAP_OPERATION_MISMATCH', 'The IMAP operation does not match its immutable target.', 'configuration', false);
         if (!permanently) {
             return failure('IMAP_SOFT_DELETE_UNSUPPORTED', 'This IMAP Provider cannot move messages to Trash. Delete them in your mail client instead.', 'configuration', false);
         }
         try {
             throwIfAborted(signal);
-            const locator = requireMessageLocator(providerMessageId);
             const client = await this.imap();
-            await client.mailboxOpen(locator.folder);
-            await client.messageDelete(locator.uid, { uid: true });
+            // Without UIDPLUS, ImapFlow falls back to a mailbox-wide EXPUNGE.
+            if (!client.capabilities?.has('UIDPLUS'))
+                return failure('IMAP_UIDPLUS_REQUIRED', 'This IMAP server cannot expunge a single message (UIDPLUS is missing). Delete it in your mail client instead.', 'configuration', false);
+            if (!this.locations) {
+                const locator = requireMessageLocator(providerMessageId);
+                const selected = await client.mailboxOpen(locator.folder);
+                if (String(selected.uidValidity) !== locator.uidValidity)
+                    return this.copyChanged();
+                const target = await client.fetchOne(locator.uid, { uid: true }, { uid: true });
+                if (!target || target.uid !== locator.uid)
+                    return this.copyChanged();
+                if (!(await client.messageFlagsAdd(String(locator.uid), ['\\Deleted'], {
+                    uid: true,
+                })))
+                    throw new Error('The IMAP server did not set the deletion flag.');
+                if (!client.capabilities.has('UIDPLUS'))
+                    throw new Error('UIDPLUS became unavailable before expunging.');
+                await this.protocol.expungeUid(client, locator.uid);
+                const remaining = await client.search({ uid: String(locator.uid) }, { uid: true });
+                if (!Array.isArray(remaining) || remaining.includes(locator.uid))
+                    throw new Error('The IMAP server still lists the deleted message.');
+                return { ok: true, value: undefined };
+            }
+            const locations = await this.locations.listValidLocations(providerMessageId);
+            if (locations.length === 0)
+                return failure('IMAP_MESSAGE_NOT_FOUND', 'No verified IMAP copy of this message is available.', 'provider', false);
+            for (const location of locations) {
+                throwIfAborted(signal);
+                if (!(await this.openCopy(location, { uid: true }, false)))
+                    return this.copyChanged();
+                const uid = String(location.uid);
+                if (!(await client.messageFlagsAdd(uid, ['\\Deleted'], { uid: true })))
+                    throw new Error('The IMAP server did not set the deletion flag.');
+                if (!client.capabilities.has('UIDPLUS'))
+                    throw new Error('UIDPLUS became unavailable before expunging.');
+                // Never use messageDelete(): its post-STORE capability check can choose global EXPUNGE.
+                await this.protocol.expungeUid(client, location.uid);
+                const remaining = await client.search({ uid }, { uid: true });
+                if (!Array.isArray(remaining) || remaining.includes(location.uid))
+                    throw new Error('The IMAP server still lists the deleted message.');
+                // Earlier copies stay committed if a later copy fails.
+                await this.locations.commitDeleted(location);
+            }
             return { ok: true, value: undefined };
         }
         catch (error) {
@@ -532,10 +734,15 @@ export class ImapSmtpAdapter {
             return { ok: false, error: classifyError(error, 'IMAP_UPDATE_FLAGS') };
         }
     }
+    closeImap() {
+        const client = this.imapClient;
+        this.imapClient = undefined;
+        client?.close();
+    }
     async imap() {
         if (this.imapClient)
             return this.imapClient;
-        const client = createImapClient(this.config.imap, this.credential);
+        const client = this.protocol.imap(this.config.imap, this.credential);
         await client.connect();
         this.imapClient = client;
         return client;
@@ -642,6 +849,9 @@ export class ImapSmtpAdapter {
     async fetchMessages(folder, uidValidity, uids, signal) {
         if (uids.length === 0)
             return [];
+        const observed = this.locations
+            ? await this.locations.listKnownLocations()
+            : [];
         const metadata = [];
         const client = await this.imap();
         // Finish FETCH before issuing more IMAP commands: nested downloads deadlock.
@@ -654,12 +864,450 @@ export class ImapSmtpAdapter {
         const messages = [];
         for (const message of metadata) {
             throwIfAborted(signal);
-            messages.push(await this.normalizeMessage(folder, uidValidity, message, signal));
+            const normalized = await this.normalizeMessage(folder, uidValidity, message, signal);
+            messages.push({
+                message: normalized,
+                tuple: { folder, uidValidity, uid: message.uid },
+                observed: observed.find((location) => location.folder === folder &&
+                    location.uidValidity === uidValidity &&
+                    location.uid === message.uid),
+                emailId: message.emailId,
+                rfcMessageId: validRfcMessageId(normalized.internetMessageId),
+                structure: message.bodyStructure,
+                size: message.size,
+            });
+        }
+        return messages;
+    }
+    beginStep() {
+        this.copies = [];
+        this.changes = [];
+        this.cleanup = false;
+        this.verificationIncomplete = false;
+        this.verificationCopies.length = 0;
+        this.rawBudget = MAX_IMAP_RAW_COMPARE_BUDGET;
+        this.rawCache.clear();
+    }
+    /**
+     * Choose each copy's identity: its persisted location first, then the
+     * server's native ID, then an RFC Message-ID proven by identical MIME bytes,
+     * and otherwise the copy's own location.
+     */
+    async assignIdentities(fetched, signal) {
+        if (!this.locations)
+            return fetched.map((copy) => copy.message);
+        const stored = new Map((await this.locations.findLocations(fetched.map((copy) => copy.tuple))).map((location) => [tupleKey(location), location]));
+        const nativeSource = this.nativeSource();
+        const candidates = new Set();
+        for (const copy of fetched) {
+            if (stored.has(tupleKey(copy.tuple)))
+                continue;
+            const id = validNativeId(nativeSource, copy.emailId) ?? copy.rfcMessageId;
+            if (id)
+                candidates.add(id);
+        }
+        const sources = await this.locations.findMessageSources([...candidates]);
+        const claimed = new Map();
+        const messages = [];
+        for (const copy of fetched) {
+            throwIfAborted(signal);
+            const existing = stored.get(tupleKey(copy.tuple));
+            const locationId = existing?.id ?? randomUUID();
+            const nativeId = validNativeId(nativeSource, copy.emailId);
+            let identity;
+            if (existing) {
+                if (nativeId &&
+                    (existing.source === 'objectid' ||
+                        existing.source === 'x-gm-msgid') &&
+                    (existing.source !== nativeSource ||
+                        existing.providerMessageId !== nativeId)) {
+                    throw Object.assign(new Error('The IMAP native identity conflicts with its stored copy.'), { code: 'IMAP_IDENTITY_CONFLICT' });
+                }
+                // A verified location keeps its identity even if a native field is missing now.
+                identity = {
+                    id: existing.providerMessageId,
+                    source: existing.source ?? 'locator',
+                };
+            }
+            else if (nativeId && nativeSource) {
+                if (sources.has(nativeId) && sources.get(nativeId) !== nativeSource) {
+                    throw Object.assign(new Error('The IMAP identity source conflicts with a stored message.'), { code: 'IMAP_IDENTITY_CONFLICT' });
+                }
+                if (!sources.has(nativeId) || sources.get(nativeId) === nativeSource)
+                    identity = { id: nativeId, source: nativeSource };
+            }
+            else if (copy.rfcMessageId) {
+                const id = copy.rfcMessageId;
+                if (sources.has(id) &&
+                    sources.get(id) !== undefined &&
+                    sources.get(id) !== 'rfc-message-id') {
+                    throw Object.assign(new Error('The IMAP identity source conflicts with a stored message.'), { code: 'IMAP_IDENTITY_CONFLICT' });
+                }
+                const claimer = claimed.get(id);
+                if (claimer
+                    ? await this.sameRawCopies(claimer, copy, signal)
+                    : !sources.has(id) ||
+                        (sources.get(id) === 'rfc-message-id' &&
+                            (await this.matchesStoredCopy(id, copy, signal)))) {
+                    identity = { id, source: 'rfc-message-id' };
+                    if (!claimer)
+                        claimed.set(id, copy);
+                }
+            }
+            identity ??= {
+                id: encodeCopyLocator(copy.tuple, locationId),
+                source: 'locator',
+            };
+            this.copies.push({
+                ...copy.tuple,
+                observed: copy.observed,
+                locationId,
+                providerMessageId: identity.id,
+                source: identity.source,
+                read: copy.message.read,
+                starred: copy.message.starred,
+                receivedAt: copy.message.receivedAt,
+            });
+            messages.push({ ...copy.message, providerMessageId: identity.id });
         }
         return messages;
     }
+    nativeSource() {
+        const capabilities = this.imapClient?.capabilities;
+        if (capabilities?.has('OBJECTID'))
+            return 'objectid';
+        if (capabilities?.has('X-GM-EXT-1'))
+            return 'x-gm-msgid';
+        return undefined;
+    }
+    /** Compare a new copy with a verified copy already stored under this RFC Message-ID. */
+    async matchesStoredCopy(id, copy, signal) {
+        for (const location of await this.locations.listValidLocations(id)) {
+            if (location.source !== 'rfc-message-id')
+                return false;
+            const stored = await this.openCopy(location, {
+                size: true,
+                bodyStructure: true,
+            });
+            if (!stored)
+                continue;
+            return this.sameRawCopies({
+                message: copy.message,
+                tuple: location,
+                structure: stored.bodyStructure,
+                size: stored.size,
+            }, copy, signal);
+        }
+        // A candidate that is gone, pending or unreadable cannot prove anything.
+        return false;
+    }
+    async sameRawCopies(left, right, signal) {
+        const first = await this.rawCopy(left, signal);
+        const second = first && (await this.rawCopy(right, signal));
+        return Boolean(first && second && first.equals(second));
+    }
+    /** Complete raw MIME of one copy within the per-message and per-step budgets. */
+    async rawCopy(copy, signal) {
+        const key = tupleKey(copy.tuple);
+        if (this.rawCache.has(key))
+            return this.rawCache.get(key);
+        let raw;
+        if (rawComparable(copy.structure, copy.size, MAX_IMAP_RAW_COMPARE_BYTES) &&
+            copy.size + 1 <= this.rawBudget) {
+            // The probe byte past the reported size is charged as well.
+            this.rawBudget -= copy.size + 1;
+            try {
+                const client = await this.imap();
+                const mailbox = await client.mailboxOpen(copy.tuple.folder, {
+                    readOnly: true,
+                });
+                raw =
+                    String(mailbox.uidValidity) === copy.tuple.uidValidity
+                        ? await readRawSource(client, copy.tuple.uid, copy.size, signal)
+                        : undefined;
+            }
+            catch (error) {
+                const classified = classifyError(error, 'IMAP_READ_SOURCE');
+                if (classified.retryable || classified.category === 'authentication')
+                    throw error;
+                raw = undefined;
+            }
+        }
+        this.rawCache.set(key, raw);
+        return raw;
+    }
+    /**
+     * Select a stored copy and confirm it is still the same message. Returns
+     * undefined for a changed generation, a missing UID or another identity.
+     */
+    async openCopy(location, query, readOnly = true) {
+        try {
+            const client = await this.imap();
+            const mailbox = await client.mailboxOpen(location.folder, { readOnly });
+            if (String(mailbox.uidValidity) !== location.uidValidity)
+                return undefined;
+            const message = await client.fetchOne(location.uid, { ...query, uid: true, envelope: true }, { uid: true });
+            if (!message || message.uid !== location.uid)
+                return undefined;
+            const source = location.source ?? 'locator';
+            if (source === 'objectid' || source === 'x-gm-msgid') {
+                const currentSource = this.nativeSource();
+                const nativeId = validNativeId(currentSource, message.emailId);
+                // A UID cannot identify another message within the same UIDVALIDITY.
+                // Missing native metadata preserves this previously verified tuple.
+                return !nativeId ||
+                    (currentSource === source && nativeId === location.providerMessageId)
+                    ? message
+                    : undefined;
+            }
+            if (source === 'rfc-message-id')
+                return message.envelope?.messageId === location.providerMessageId
+                    ? message
+                    : undefined;
+            return source === 'locator' &&
+                encodeCopyLocator(location, location.id) === location.providerMessageId
+                ? message
+                : undefined;
+        }
+        catch (error) {
+            const classified = classifyError(error, 'IMAP_OPEN_COPY');
+            if (classified.retryable || classified.category === 'authentication')
+                throw error;
+            return undefined;
+        }
+    }
+    copyChanged() {
+        return failure('IMAP_MESSAGE_COPY_CHANGED', 'An IMAP copy of this message changed. Synchronize the mailbox and try again.', 'provider', false);
+    }
+    /** Change one flag on every verified copy, committing each confirmed copy on its own. */
+    async updateCopyFlags(providerMessageId, flag, enabled, signal) {
+        try {
+            throwIfAborted(signal);
+            const locations = await this.locations.listValidLocations(providerMessageId);
+            if (locations.length === 0)
+                return failure('IMAP_MESSAGE_NOT_FOUND', 'No verified IMAP copy of this message is available.', 'provider', false);
+            const client = await this.imap();
+            for (const location of locations) {
+                throwIfAborted(signal);
+                if (!(await this.openCopy(location, { uid: true }, false)))
+                    return this.copyChanged();
+                const uid = String(location.uid);
+                const method = enabled ? 'messageFlagsAdd' : 'messageFlagsRemove';
+                if (!(await client[method](uid, [flag], { uid: true })))
+                    throw new Error('The IMAP server did not update the message flags.');
+                const confirmed = await client.fetchOne(uid, { uid: true, flags: true }, { uid: true });
+                if (!confirmed ||
+                    confirmed.uid !== location.uid ||
+                    !confirmed.flags ||
+                    confirmed.flags.has(flag) !== enabled)
+                    throw new Error('The IMAP server did not confirm the message flags.');
+                await this.locations.commitFlags(location, flag === '\\Seen' ? { read: enabled } : { starred: enabled });
+            }
+            return { ok: true, value: undefined };
+        }
+        catch (error) {
+            return { ok: false, error: classifyError(error, 'IMAP_UPDATE_FLAGS') };
+        }
+    }
+    /**
+     * One complete pass over every selectable folder. It reconciles stored
+     * copies, and two matching passes in a row authorize orphan cleanup.
+     */
+    async verifyFolders(mailboxes, folders, previous, signal) {
+        const attempts = (previous?.attempts ?? 0) + 1;
+        const snapshot = {};
+        const present = new Map();
+        const excluded = new Set(previous?.excluded ?? []);
+        const uncertainGenerations = new Set();
+        let stable = true;
+        const known = await this.locations.listKnownLocations();
+        for (const mailbox of mailboxes) {
+            throwIfAborted(signal);
+            const client = await this.imap();
+            const selected = await client.mailboxOpen(mailbox.path, {
+                readOnly: true,
+            });
+            const state = await this.selectedFolderState(selected);
+            const cursor = folders[mailbox.path];
+            if (state.uidValidity !== cursor.uidValidity)
+                return 'generation-changed';
+            // New mail arrived after this page: import it before verifying again.
+            if (state.uidNext !== cursor.uidNext)
+                stable = false;
+            let uids;
+            try {
+                uids = await client.search({ all: true }, { uid: true });
+            }
+            catch (error) {
+                if (!isResponseLimitError(error))
+                    throw error;
+                // The UID list exceeds the protocol budget: keep every copy and skip cleanup.
+                this.closeImap();
+                this.verificationIncomplete = true;
+                return { done: true, verify: { attempts, excluded: [...excluded] } };
+            }
+            if (!Array.isArray(uids))
+                // Retry the step: an incomplete SEARCH proves nothing about missing copies.
+                throw Object.assign(new Error(`Could not search ${mailbox.path} during verification.`), { code: 'IMAP_SEARCH_FAILED', serverResponseCode: 'UNAVAILABLE' });
+            if (uids.some((uid) => !Number.isSafeInteger(uid) || uid < 1 || uid > 4294967295))
+                throw new Error('Invalid UID in IMAP SEARCH.');
+            present.set(mailbox.path, new Set(uids));
+            snapshot[mailbox.path] = { ...state, uids: compressUids(uids) };
+            if (Buffer.byteLength(JSON.stringify(snapshot)) >
+                MAX_IMAP_VERIFY_STATE_BYTES) {
+                this.verificationIncomplete = true;
+                return { done: true, verify: { attempts, excluded: [...excluded] } };
+            }
+            const bound = new Set(known
+                .filter((location) => location.folder === mailbox.path &&
+                location.uidValidity === state.uidValidity &&
+                location.status === 'valid')
+                .map((location) => location.uid));
+            const unbound = uids.filter((uid) => !bound.has(uid) &&
+                !excluded.has(tupleKey({
+                    folder: mailbox.path,
+                    uidValidity: state.uidValidity,
+                    uid,
+                })));
+            if (unbound.length) {
+                // Reconcile copies below the scan high-water mark, including old-date moves.
+                const fetched = await this.fetchMessages(mailbox.path, state.uidValidity, unbound.slice(0, this.syncRun?.policy.batchSize ?? 100), signal);
+                const sources = await this.locations.findMessageSources(fetched.flatMap((copy) => {
+                    const id = validNativeId(this.nativeSource(), copy.emailId) ??
+                        copy.rfcMessageId;
+                    return id ? [id] : [];
+                }));
+                for (const copy of fetched) {
+                    const id = validNativeId(this.nativeSource(), copy.emailId) ??
+                        copy.rfcMessageId;
+                    const beforeImportDate = this.syncRun?.policy.receivedAfter &&
+                        copy.message.receivedAt &&
+                        Date.parse(copy.message.receivedAt) <
+                            Date.parse(this.syncRun.policy.receivedAfter);
+                    if (beforeImportDate && !copy.observed && (!id || !sources.has(id)))
+                        excluded.add(tupleKey(copy.tuple));
+                    else
+                        this.verificationCopies.push(copy);
+                }
+                stable = false;
+            }
+        }
+        const after = this.selectableMailboxes(await this.listMailboxes());
+        if (after.length !== mailboxes.length ||
+            after.some((box) => !snapshot[box.path]))
+            stable = false;
+        for (const mailbox of after) {
+            const selected = await (await this.imap()).mailboxOpen(mailbox.path, { readOnly: true });
+            const state = await this.selectedFolderState(selected);
+            const rechecked = await (await this.imap()).search({ all: true }, { uid: true });
+            if (!Array.isArray(rechecked))
+                throw new Error('The final IMAP SEARCH was incomplete.');
+            if (compressUids(rechecked) !== snapshot[mailbox.path]?.uids)
+                stable = false;
+            if (state.uidValidity !== snapshot[mailbox.path]?.uidValidity ||
+                state.uidNext !== snapshot[mailbox.path]?.uidNext)
+                stable = false;
+            if (state.uidValidity !== snapshot[mailbox.path]?.uidValidity)
+                uncertainGenerations.add(mailbox.path);
+        }
+        const finalRange = this.selectableMailboxes(await this.listMailboxes());
+        const finalPaths = new Set(finalRange.map((mailbox) => mailbox.path));
+        if (finalPaths.size !== mailboxes.length ||
+            mailboxes.some((mailbox) => !finalPaths.has(mailbox.path)))
+            stable = false;
+        for (const mailbox of finalRange) {
+            const generation = mailbox.status?.uidValidity;
+            const uidNext = mailbox.status?.uidNext;
+            if (generation !== undefined &&
+                String(generation) !== snapshot[mailbox.path]?.uidValidity) {
+                uncertainGenerations.add(mailbox.path);
+                stable = false;
+            }
+            if (uidNext !== undefined && uidNext !== snapshot[mailbox.path]?.uidNext)
+                stable = false;
+        }
+        for (const location of known) {
+            if (this.verificationCopies.some((copy) => tupleKey(copy.tuple) === tupleKey(location)))
+                continue;
+            const uids = present.get(location.folder);
+            const generation = snapshot[location.folder]?.uidValidity;
+            const replaced = generation !== location.uidValidity &&
+                (location.source === 'objectid' || location.source === 'x-gm-msgid') &&
+                known.some((copy) => copy.messageId === location.messageId &&
+                    copy.source === location.source &&
+                    copy.providerMessageId === location.providerMessageId &&
+                    copy.status === 'valid' &&
+                    snapshot[copy.folder]?.uidValidity === copy.uidValidity &&
+                    present.get(copy.folder)?.has(copy.uid));
+            if (stable &&
+                location.status === 'valid' &&
+                location.source &&
+                generation === location.uidValidity &&
+                uids?.has(location.uid)) {
+                const observed = await this.openCopy(location, { flags: true });
+                if (!observed?.flags)
+                    throw new Error('The IMAP copy identity or flags changed during verification.');
+                this.copies.push({
+                    flagsOnly: true,
+                    locationId: location.id,
+                    observed: location,
+                    folder: location.folder,
+                    uidValidity: location.uidValidity,
+                    uid: location.uid,
+                    providerMessageId: location.providerMessageId,
+                    source: location.source,
+                    read: observed.flags.has('\\Seen'),
+                    starred: observed.flags.has('\\Flagged'),
+                });
+            }
+            const next = !finalPaths.has(location.folder)
+                ? { status: 'pending', reason: 'folder-unavailable' }
+                : uncertainGenerations.has(location.folder)
+                    ? { status: 'pending', reason: 'generation-changed' }
+                    : replaced && stable
+                        ? { status: 'invalid', reason: 'generation-replaced' }
+                        : !uids
+                            ? { status: 'pending', reason: 'folder-unavailable' }
+                            : generation !== location.uidValidity
+                                ? { status: 'pending', reason: 'generation-changed' }
+                                : uids.has(location.uid)
+                                    ? { status: 'valid' }
+                                    : { status: 'invalid', reason: 'uid-missing' };
+            if (next.status !== 'valid' || next.status !== location.status)
+                this.changes.push({ id: location.id, observed: location, ...next });
+        }
+        if (Buffer.byteLength(JSON.stringify({ snapshot, excluded: [...excluded] })) >
+            MAX_IMAP_VERIFY_STATE_BYTES) {
+            this.verificationIncomplete = true;
+            return { done: true, verify: { attempts } };
+        }
+        if (!stable) {
+            if (attempts >= MAX_IMAP_VERIFY_ATTEMPTS) {
+                this.verificationIncomplete = true;
+                return { done: true, verify: { attempts, excluded: [...excluded] } };
+            }
+            return { done: false, verify: { attempts, excluded: [...excluded] } };
+        }
+        if (previous?.folders && sameSnapshot(previous.folders, snapshot)) {
+            this.cleanup = true;
+            return { done: true };
+        }
+        if (attempts >= MAX_IMAP_VERIFY_ATTEMPTS) {
+            this.verificationIncomplete = true;
+            return { done: true, verify: { attempts, excluded: [...excluded] } };
+        }
+        return {
+            done: false,
+            verify: { attempts, folders: snapshot, excluded: [...excluded] },
+        };
+    }
     async normalizeMessage(folder, uidValidity, message, signal, maxBodyBytes = MAX_IMAP_BODY_BYTES) {
-        const header = message.headers?.subarray(0, 64 * 1024);
+        const header = message.headers;
+        if (!header ||
+            header.byteLength > MAX_HEADER_BYTES ||
+            !header.subarray(-4).equals(Buffer.from('\r\n\r\n'))) {
+            throw Object.assign(new Error('The complete IMAP header could not be read within the header budget.'), { code: 'IMAP_HEADER_INCOMPLETE' });
+        }
         const parsed = header
             ? await simpleParser(Buffer.concat([header, Buffer.from('\r\n\r\n')]), {
                 skipHtmlToText: true,
@@ -698,7 +1346,8 @@ export class ImapSmtpAdapter {
             providerMessageId,
             ...body,
             size: message.size,
-            internetMessageId: parsed?.messageId ?? message.envelope?.messageId,
+            // Preserve the header value; mailparser can wrap malformed bare IDs in brackets.
+            internetMessageId: /^message-id:[\t ]*([^\r\n]*(?:\r\n[\t ][^\r\n]*)*)\r\n/im.exec(header.toString('utf8'))?.[1],
             providerFolderIds: [folder],
             from,
             to,
@@ -718,4 +1367,46 @@ export class ImapSmtpAdapter {
         };
     }
 }
+function tupleKey(tuple) {
+    return JSON.stringify([tuple.folder, tuple.uidValidity, tuple.uid]);
+}
+const MAX_UINT64 = 2n ** 64n - 1n;
+/** The server's own object ID, never ImapFlow's synthesized message.id. */
+function validNativeId(source, value) {
+    if (!value)
+        return undefined;
+    if (source === 'objectid')
+        return /^[A-Za-z0-9_-]{1,255}$/.test(value) ? value : undefined;
+    if (source === 'x-gm-msgid')
+        // Keep the decimal string: a JavaScript number would lose uint64 precision.
+        return /^(0|[1-9]\d{0,19})$/.test(value) && BigInt(value) <= MAX_UINT64
+            ? value
+            : undefined;
+    return undefined;
+}
+function validRfcMessageId(value) {
+    return value &&
+        value.length <= MAX_IMAP_PROVIDER_ID_LENGTH &&
+        /^<[\x21-\x3b\x3d\x3f-\x7e]+@[\x21-\x3b\x3d\x3f-\x7e]+>$/.test(value)
+        ? value
+        : undefined;
+}
+function sameSnapshot(left, right) {
+    const paths = Object.keys(right);
+    return (Object.keys(left).length === paths.length &&
+        paths.every((path) => left[path]?.uidValidity === right[path].uidValidity &&
+            left[path].uidNext === right[path].uidNext &&
+            left[path].uids === right[path].uids));
+}
+function isResponseLimitError(error) {
+    return ['LiteralTooLarge', 'ResponseTooLarge', 'LineTooLarge'].includes(error?.code ?? '');
+}
+/** ImapFlow 2.2.10's command primitive keeps the verb explicitly scoped to a UID. */
+async function expungeUid(client, uid) {
+    const connection = client;
+    const response = await connection.exec('UID EXPUNGE', [
+        { type: 'SEQUENCE', value: String(uid) },
+    ]);
+    response.next();
+}
 //# sourceMappingURL=adapter.js.map