@wirecat/cli-core 0.487.0 → 0.488.0

package/CHANGELOG.md

@@ -7,6 +7,17 @@ shape of this file.
 
 ## Unreleased
 
+## 0.488.0 — 30.10.2026
+
+### Added
+
+- **A quick mail sync proves deletions and refreshes flags again (store version 39).** The store keeps each email's
+  place on an IMAP server (`email_locations`: folder, UIDVALIDITY, UID), saved with the email, so a run that lists only
+  UIDs and flags can tell which stored mail the server dropped and which flags changed, without reading the mail. The
+  first run on mail stored before this lists the folder once with headers to record the places; after that the folder
+  is quick. A new UIDVALIDITY drops the folder's places. New `store.mail.folderEmails`, `setLocations` and
+  `forgetLocations`, and `locations` on `EmailInput`.
+
 ## 0.487.0 — 30.10.2026
 
 ### Added

package/dist/mail/attachments.d.ts

@@ -0,0 +1,15 @@
+import type { LoadEngine } from "../readers/types.js";
+import type { AttachmentInput, EmailAddress } from "../store/index.js";
+import type { Address, ParsedMail } from "./parse.js";
+import type { MailSyncResult } from "./sync.js";
+export declare const MAX_ATTACHMENT_CHARS = 200000;
+/** One chunk of raw mail is held in memory at a time. */
+export declare const FETCH_CHUNK = 50;
+export declare const addressOf: ({ email, name }: Address) => EmailAddress;
+export type AttachmentLog = NonNullable<MailSyncResult["attachments"]>;
+/** A parsed email's attachments as the store keeps them, each with its text or why it has none. */
+export declare const readAttachments: (mail: ParsedMail, messageId: string, { loadEngine, maxAttachmentChars, at }: {
+    loadEngine: LoadEngine;
+    maxAttachmentChars: number;
+    at: number;
+}, log: AttachmentLog) => Promise<AttachmentInput[]>;

package/dist/mail/attachments.js

@@ -0,0 +1,38 @@
+import { extractText } from "../attachments/extract.js";
+export const MAX_ATTACHMENT_CHARS = 200_000;
+/** One chunk of raw mail is held in memory at a time. */
+export const FETCH_CHUNK = 50;
+export const addressOf = ({ email, name }) => ({ address: email, name });
+/** A parsed email's attachments as the store keeps them, each with its text or why it has none. */
+export const readAttachments = async (mail, messageId, { loadEngine, maxAttachmentChars, at }, log) => {
+    const files = [];
+    for (const [position, part] of mail.attachments.entries()) {
+        const extraction = part.bytes === null
+            ? { status: "unavailable" }
+            : await extractText(part.bytes, { kind: "file", name: part.name, mime: part.mime, path: part.name }, loadEngine);
+        log.push({ message: messageId, part: part.id, status: extraction.status });
+        const extracted = extraction.status === "extracted" ? extraction : undefined;
+        files.push({
+            position,
+            kind: "file",
+            mime: part.mime,
+            name: part.name,
+            title: null,
+            url: null,
+            size: part.bytes === null ? null : part.bytes.byteLength,
+            width: null,
+            height: null,
+            duration: null,
+            providerRef: { message: messageId, part: part.id },
+            localPath: null,
+            text: extracted ? extracted.text.slice(0, maxAttachmentChars) : null,
+            normalizedText: null,
+            extraction: extracted ? "text" : "failed",
+            extractor: extracted ? extracted.extractor : null,
+            extractionError: extracted ? null : extraction.status,
+            contentSha256: null,
+            extractedAt: at,
+        });
+    }
+    return files;
+};

package/dist/mail/locations.d.ts

@@ -0,0 +1,28 @@
+import type { FolderEmail, MessageStore } from "../store/index.js";
+import type { Quick, Searched } from "./source.js";
+/** The flags a listing shows that differ from the stored ones, by Message-ID, for mail whose UID is stored. */
+export declare const flagsToRefresh: (rows: ReadonlyMap<string, FolderEmail>, listing: readonly Quick[]) => Map<string, {
+    read?: boolean;
+    flagged?: boolean;
+}>;
+export interface LocationProof {
+    /** Each folder listed by UID: what the server searched, and the stored mail with a UID there, before this run saved. */
+    folders: ReadonlyMap<string, {
+        searched: Searched;
+        rows: readonly FolderEmail[];
+    }>;
+    /** Mail received before this is outside the window IMAP's SINCE can vouch for. */
+    windowFrom: number;
+    /** Messages this run saw in any folder: one moved to another folder is not gone. */
+    present: ReadonlySet<string>;
+    now: number;
+}
+/**
+ * A folder listed by UID proves a deletion from the UIDs it stores: mail in the window whose UID the server no
+ * longer lists is gone from that folder, and gone altogether when no other location of it is left. Nothing is
+ * changed when too many would go, as for a header listing.
+ */
+export declare const proveByLocation: (store: MessageStore, accountId: number, { folders, windowFrom, present, now }: LocationProof, refuse: (gone: number, candidates: number) => string | undefined) => Promise<{
+    deleted: number;
+    skipped?: string;
+}>;

package/dist/mail/locations.js

@@ -0,0 +1,54 @@
+/** The flags a listing shows that differ from the stored ones, by Message-ID, for mail whose UID is stored. */
+export const flagsToRefresh = (rows, listing) => {
+    const changes = new Map();
+    for (const { uid, seen, flagged } of listing) {
+        const row = rows.get(uid);
+        if (row === undefined)
+            continue;
+        const changed = {
+            ...(seen === undefined || seen === row.read ? {} : { read: seen }),
+            ...(flagged === undefined || flagged === row.flagged ? {} : { flagged }),
+        };
+        if (Object.keys(changed).length)
+            changes.set(row.externalId, changed);
+    }
+    return changes;
+};
+/**
+ * A folder listed by UID proves a deletion from the UIDs it stores: mail in the window whose UID the server no
+ * longer lists is gone from that folder, and gone altogether when no other location of it is left. Nothing is
+ * changed when too many would go, as for a header listing.
+ */
+export const proveByLocation = async (store, accountId, { folders, windowFrom, present, now }, refuse) => {
+    const candidates = new Set();
+    const lost = new Map();
+    const lostUids = new Map();
+    for (const [folder, { searched, rows }] of folders) {
+        const inWindow = new Set(searched.uids.map(Number));
+        for (const row of rows) {
+            if (row.receivedAt === null || row.receivedAt < windowFrom)
+                continue;
+            candidates.add(row.externalId);
+            if (inWindow.has(row.uid))
+                continue;
+            const entry = lost.get(row.externalId) ?? { row, locations: 0, folders: new Set() };
+            entry.locations++;
+            entry.folders.add(folder);
+            lost.set(row.externalId, entry);
+            const uids = lostUids.get(folder) ?? [];
+            uids.push(row.uid);
+            lostUids.set(folder, uids);
+        }
+    }
+    const gone = [...lost]
+        .filter(([id, entry]) => !present.has(id) && entry.locations === entry.row.locations && entry.folders.size >= entry.row.folders)
+        .map(([id]) => id);
+    const refused = refuse(gone.length, candidates.size);
+    if (refused)
+        return { deleted: 0, skipped: refused };
+    // Marked first: an interruption leaves locations that prove it again, never an email with no way to be found.
+    const deleted = gone.length ? await store.mail.markDeleted(accountId, gone, now) : 0;
+    for (const [folder, uids] of lostUids)
+        await store.mail.forgetLocations(accountId, folder, folders.get(folder).searched.validity, uids);
+    return { deleted };
+};

package/dist/mail/quick.d.ts

@@ -20,6 +20,8 @@ export interface QuickPlan {
     asked: number;
 }
 export interface QuickRead {
+    /** Every message of the window the server listed, done or not: its flags refresh the stored mail. */
+    listing: Quick[];
     /** Messages read and identified, with their raw bytes so the save does not read them again. */
     read: {
         item: Listed;
@@ -36,4 +38,4 @@ export interface QuickRead {
  * One round of a folder whose server gives no cheap message id: UID, flags, date and size of what is not done, then
  * the raw bytes of the newest of it, whose Message-ID gives the id the full listing would have given.
  */
-export declare const readQuick: (source: MailSource & Required<Pick<MailSource, "quick" | "identify">>, since: Date, folder: string, searched: Searched, done: Ranges, max: number, plan: QuickPlan | undefined) => Promise<QuickRead>;
+export declare const readQuick: (source: MailSource & Required<Pick<MailSource, "quick" | "identify">>, since: Date, folder: string, searched: Searched, done: Ranges, max: number, plan: QuickPlan | undefined, located: ReadonlySet<string>) => Promise<QuickRead>;

package/dist/mail/quick.js

@@ -6,16 +6,18 @@ export const keyOf = (folder, uid) => `${folder}\0${uid}`;
  * One round of a folder whose server gives no cheap message id: UID, flags, date and size of what is not done, then
  * the raw bytes of the newest of it, whose Message-ID gives the id the full listing would have given.
  */
-export const readQuick = async (source, since, folder, searched, done, max, plan) => {
-    const answer = { read: [], unreadable: [], capped: false };
+export const readQuick = async (source, since, folder, searched, done, max, plan, located) => {
+    const answer = { listing: [], read: [], unreadable: [], capped: false };
     let items = plan?.quick.get(folder);
     if (items === undefined) {
-        const asked = searched.uids.filter((uid) => !within(done, Number(uid)) && (plan?.only === undefined || plan.only.has(keyOf(folder, uid))));
+        // Mail whose location is known is listed for its flags only; it is never read again.
+        const asked = searched.uids.filter((uid) => plan?.only === undefined || plan.only.has(keyOf(folder, uid)));
+        const wanted = asked.filter((uid) => !within(done, Number(uid)) && !located.has(uid));
         if (asked.length > 0 && plan) {
             const stop = plan.budget.take();
             if (stop) {
                 plan.stopped = stop;
-                plan.remaining += asked.length;
+                plan.remaining += wanted.length;
                 return { ...answer, capped: true };
             }
         }
@@ -23,15 +25,16 @@ export const readQuick = async (source, since, folder, searched, done, max, plan
         if (plan !== undefined) {
             plan.quick.set(folder, items);
             const got = new Set(items.map(({ uid }) => uid));
-            for (const uid of asked)
+            for (const uid of wanted)
                 if (!got.has(uid)) {
                     plan.failed.add(keyOf(folder, uid));
                     plan.unlisted.add(keyOf(folder, uid));
                 }
         }
     }
+    answer.listing = items;
     const open = items
-        .filter(({ uid }) => !within(done, Number(uid)) && !plan?.tried.has(keyOf(folder, uid)))
+        .filter(({ uid }) => !within(done, Number(uid)) && !located.has(uid) && !plan?.tried.has(keyOf(folder, uid)))
         .sort((a, b) => Number(b.uid) - Number(a.uid));
     const slice = open.slice(0, plan === undefined ? max : (source.fetchBatch ?? FETCH_SLICE));
     if (plan !== undefined)

package/dist/mail/resume.d.ts

@@ -30,6 +30,12 @@ export interface ResumePoint {
         total: number;
         at: string;
     };
+    /**
+     * From this date on, every stored email of the folder has its UID in `email_locations`: a listing that gives UIDs
+     * and flags is enough to refresh flags and prove deletions. Absent after an engine that did not know locations
+     * wrote the point, and then one header listing records them.
+     */
+    located?: string;
     /** What the latest run listed, whatever its window. */
     last?: {
         since: string;

package/dist/mail/retry.d.ts

@@ -0,0 +1,23 @@
+import type { LoadEngine } from "../readers/types.js";
+import type { MessageStore } from "../store/index.js";
+import type { MailSource } from "./source.js";
+import type { MailAccount, MailSyncOptions, MailSyncResult } from "./sync.js";
+/** Failures a newer install mends: the reader was missing, not the file at fault, so reading again can succeed. */
+export declare const RETRYABLE_ATTACHMENTS: string[];
+/**
+ * Reads again, by Gmail id, only the stored emails with an attachment a missing reader left unread: memo keeps no
+ * attachment file, so the email comes from the source again. Nothing is listed, so no deletion is proved and no
+ * resume point moves; the email keeps its stored mailboxes, flags and rule.
+ */
+export declare const retryUnread: ({ store, source, account, since, max, maxBodyChars, maxAttachmentChars, now, loadEngine, index, }: {
+    store: MessageStore;
+    source: MailSource;
+    account: MailAccount;
+    since: Date;
+    max: number;
+    maxBodyChars: number;
+    maxAttachmentChars: number;
+    now: () => number;
+    loadEngine: LoadEngine;
+    index: MailSyncOptions["index"];
+}) => Promise<MailSyncResult>;

package/dist/mail/retry.js

@@ -0,0 +1,110 @@
+import { CliError } from "@wirecat/cli-base";
+import { addressOf, FETCH_CHUNK, readAttachments } from "./attachments.js";
+import { mailFolders } from "./folders.js";
+import { mailHeadersOf } from "./headers.js";
+import { parseMail } from "./parse.js";
+/** Failures a newer install mends: the reader was missing, not the file at fault, so reading again can succeed. */
+export const RETRYABLE_ATTACHMENTS = ["engine-missing"];
+/**
+ * Reads again, by Gmail id, only the stored emails with an attachment a missing reader left unread: memo keeps no
+ * attachment file, so the email comes from the source again. Nothing is listed, so no deletion is proved and no
+ * resume point moves; the email keeps its stored mailboxes, flags and rule.
+ */
+export const retryUnread = async ({ store, source, account, since, max, maxBodyChars, maxAttachmentChars, now, loadEngine, index, }) => {
+    const startedAt = now();
+    const key = { provider: "email", account: account.address.toLowerCase() };
+    const find = source.find;
+    if (find === undefined)
+        throw new CliError("validation_error", "--retry-attachments reads mail again by its Gmail id; an mbox file or a plain IMAP account has none");
+    const accountId = await store.accounts.saveAccount(key, { name: account.address });
+    const wanted = (await store.mail.emailRefs({ accountId, attachmentErrors: RETRYABLE_ATTACHMENTS })).map(({ externalId }) => externalId);
+    const chosen = wanted.slice(0, max);
+    const found = new Map();
+    for (const folder of await mailFolders(account, source)) {
+        const left = chosen.filter((id) => !found.has(id));
+        if (left.length === 0)
+            break;
+        for (const item of await find(left, folder))
+            if (!found.has(item.messageId))
+                found.set(item.messageId, item);
+    }
+    const items = [...found.values()];
+    const attachments = [];
+    const unreadable = [];
+    let emails = 0;
+    let extracted = 0;
+    let stillFailed = 0;
+    for (let start = 0; start < items.length; start += FETCH_CHUNK) {
+        const chunk = items.slice(start, start + FETCH_CHUNK);
+        const raws = await source.fetchRaw(chunk);
+        for (const [index, item] of chunk.entries()) {
+            const raw = raws[index];
+            const stored = await store.mail.email(accountId, item.messageId);
+            if (stored === null)
+                continue;
+            let mail;
+            try {
+                if (raw === undefined || raw instanceof Error)
+                    throw raw ?? new Error("not read");
+                mail = await parseMail(raw, maxBodyChars);
+            }
+            catch (error) {
+                unreadable.push({ message: item.messageId, reason: error instanceof Error ? error.message : String(error) });
+                continue;
+            }
+            const before = new Map(stored.attachments.map(({ position, extractionError }) => [position, extractionError]));
+            const files = await readAttachments(mail, item.messageId, { loadEngine, maxAttachmentChars, at: startedAt }, attachments);
+            for (const file of files)
+                if (RETRYABLE_ATTACHMENTS.includes(before.get(file.position) ?? ""))
+                    if (file.extraction === "text")
+                        extracted++;
+                    else
+                        stillFailed++;
+            const header = mailHeadersOf(raw);
+            await store.mail.saveThread({
+                accountId,
+                externalId: item.threadId,
+                now: startedAt,
+                emails: [
+                    {
+                        externalId: item.messageId,
+                        subject: mail.subject,
+                        from: mail.from === null ? null : addressOf(mail.from),
+                        to: mail.to.map(addressOf),
+                        cc: mail.cc.map(addressOf),
+                        bcc: mail.bcc.map(addressOf),
+                        sentAt: null,
+                        receivedAt: Date.parse(item.receivedAt),
+                        bodyText: mail.text || null,
+                        size: item.size ?? stored.size,
+                        ...(Object.keys(header).length ? { headers: header } : {}),
+                        outgoing: mail.from === null ? null : mail.from.email === key.account,
+                        mailboxes: stored.mailboxes.map(({ externalId, name, kind }) => ({ externalId, name, kind })),
+                        ...(stored.read === null ? {} : { read: stored.read }),
+                        ...(stored.flagged === null ? {} : { flagged: stored.flagged }),
+                        metadata: stored.metadata,
+                        mailRuleId: stored.mailRuleId,
+                        attachments: files,
+                    },
+                ],
+            });
+            emails++;
+        }
+    }
+    const indexed = await index?.(store, key);
+    return {
+        account: key.account,
+        since: since.toISOString(),
+        listed: items.length,
+        saved: 0,
+        alreadyStored: 0,
+        skipped: 0,
+        envelopes: 0,
+        deleted: 0,
+        complete: wanted.length <= max && unreadable.length === 0,
+        ...(indexed === undefined ? {} : { indexed }),
+        retried: { emails, extracted, stillFailed, missing: chosen.length - items.length },
+        ...(attachments.length ? { attachments } : {}),
+        ...(unreadable.length ? { unreadable } : {}),
+    };
+};

package/dist/mail/sync.d.ts

@@ -74,7 +74,8 @@ export interface MailSyncResult {
         missing: number;
     };
 }
-export declare const MAX_ATTACHMENT_CHARS = 200000;
+export { MAX_ATTACHMENT_CHARS } from "./attachments.js";
+export { RETRYABLE_ATTACHMENTS } from "./retry.js";
 export interface MailSyncOptions {
     account: MailAccount;
     since: Date;
@@ -102,5 +103,3 @@ export interface MailSyncOptions {
     budget?: RunBudget;
 }
 export declare const syncMail: (source: MailSource, store: MessageStore, options: MailSyncOptions) => Promise<MailSyncResult>;
-/** Failures a newer install mends: the reader was missing, not the file at fault, so reading again can succeed. */
-export declare const RETRYABLE_ATTACHMENTS: string[];

package/dist/mail/sync.js

@@ -1,59 +1,27 @@
 import { CliError } from "@wirecat/cli-base";
-import { extractText, importEngine } from "../attachments/extract.js";
+import { importEngine } from "../attachments/extract.js";
+import { addressOf, FETCH_CHUNK, MAX_ATTACHMENT_CHARS, readAttachments } from "./attachments.js";
 import { BULK, bulkBox } from "./bulk.js";
 import { factsOf } from "./facts.js";
 import { GMAIL_ALL_MAIL, mailFolders } from "./folders.js";
 import { dropMissing, tooManyGone } from "./gone.js";
 import { mailHeadersOf } from "./headers.js";
+import { flagsToRefresh, proveByLocation } from "./locations.js";
 import { MAX_BODY_CHARS, parseMail } from "./parse.js";
 import { FETCH_SLICE, keyOf, readQuick } from "./quick.js";
 import { FULL_PASS_MS, MBOX_RESUME, mergeRanges, parsePoint, rangesOf, resumeKey, rulesKey, within, } from "./resume.js";
+import { retryUnread } from "./retry.js";
 import { ruleTagger, settleRuleTags } from "./rule-tags.js";
 import { decide, ENVELOPE, envelopeBox, envelopeBoxOf, tagsOf } from "./rules.js";
 const DAY = 86_400_000;
 /** IMAP's SINCE counts whole days in the server's zone, so the window's first days are not trusted. */
 const EDGE = 2 * DAY;
-export const MAX_ATTACHMENT_CHARS = 200_000;
-/** One chunk of raw mail is held in memory at a time. */
-const FETCH_CHUNK = 50;
 /** How often a long run saves where it is, so a kill costs at most this much listing again. */
 const SAVE_EVERY_MS = 30_000;
 /** About a minute of listing; the 30-day window of the timer stays below it. */
 const RESUME_FROM = 5_000;
-const addressOf = ({ email, name }) => ({ address: email, name });
-/** A parsed email's attachments as the store keeps them, each with its text or why it has none. */
-const readAttachments = async (mail, messageId, { loadEngine, maxAttachmentChars, at }, log) => {
-    const files = [];
-    for (const [position, part] of mail.attachments.entries()) {
-        const extraction = part.bytes === null
-            ? { status: "unavailable" }
-            : await extractText(part.bytes, { kind: "file", name: part.name, mime: part.mime, path: part.name }, loadEngine);
-        log.push({ message: messageId, part: part.id, status: extraction.status });
-        const extracted = extraction.status === "extracted" ? extraction : undefined;
-        files.push({
-            position,
-            kind: "file",
-            mime: part.mime,
-            name: part.name,
-            title: null,
-            url: null,
-            size: part.bytes === null ? null : part.bytes.byteLength,
-            width: null,
-            height: null,
-            duration: null,
-            providerRef: { message: messageId, part: part.id },
-            localPath: null,
-            text: extracted ? extracted.text.slice(0, maxAttachmentChars) : null,
-            normalizedText: null,
-            extraction: extracted ? "text" : "failed",
-            extractor: extracted ? extracted.extractor : null,
-            extractionError: extracted ? null : extraction.status,
-            contentSha256: null,
-            extractedAt: at,
-        });
-    }
-    return files;
-};
+export { MAX_ATTACHMENT_CHARS } from "./attachments.js";
+export { RETRYABLE_ATTACHMENTS } from "./retry.js";
 const LIST_SLICE = 200;
 export const syncMail = async (source, store, options) => {
     const { budget, index } = options;
@@ -194,6 +162,11 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
     /** Raw mail read to learn its id, kept for the save so it is not read twice. */
     const rawOf = new Map();
     const quickRan = new Set();
+    /** Folders whose server gives no cheap id: their mail's UIDs are kept, so a UID listing can stand in for headers. */
+    const locatable = new Set();
+    /** A quick folder's stored mail as it was before this run changed anything, with the UID of each. */
+    const folderMail = new Map();
+    const refreshed = new Map();
     let holding;
     const holdsMail = async (folder) => {
         holding ??= new Set((await store.mail.emailRefs({ accountId })).flatMap(({ mailboxes }) => mailboxes
@@ -234,10 +207,16 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
                 searched.validity !== undefined &&
                 source.quick !== undefined &&
                 source.identify !== undefined;
+            if (quickable)
+                locatable.add(folder);
+            // The UID listing proves deletions only for mail whose UID is stored; a folder stored before locations existed
+            // gets them from one header listing.
+            const located = point?.located !== undefined && Date.parse(point.located) <= since.getTime();
+            const quickReady = async () => (sameRules && located) || !(await holdsMail(folder));
             const inQuick = quickable &&
                 (plan === undefined
-                    ? sameRules || !(await holdsMail(folder))
-                    : plan.quickFolders.has(folder) || (!plan.modes.has(folder) && (sameRules || !(await holdsMail(folder)))));
+                    ? await quickReady()
+                    : plan.quickFolders.has(folder) || (!plan.modes.has(folder) && (await quickReady())));
             if (inQuick) {
                 plan?.quickFolders.add(folder);
                 plan?.modes.set(folder, "resumed");
@@ -246,7 +225,12 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
                 const done = point !== undefined && sameRules ? point.done : [];
                 if (point !== undefined && sameRules)
                     earlier.set(folder, point);
-                const round = await readQuick(source, since, folder, searched, done, max, plan);
+                const rows = await store.mail.folderEmails(accountId, folder, searched.validity);
+                folderMail.set(folder, rows);
+                const byUid = new Map(rows.map((row) => [String(row.uid), row]));
+                const round = await readQuick(source, since, folder, searched, done, max, plan, new Set(byUid.keys()));
+                for (const [id, flags] of flagsToRefresh(byUid, round.listing))
+                    refreshed.set(id, { ...refreshed.get(id), ...flags });
                 if (round.capped)
                     capped = true;
                 for (const { item, raw } of round.read) {
@@ -260,12 +244,13 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
             if (point !== undefined && point.validity === searched.validity && point.rules === ruleSet) {
                 earlier.set(folder, point);
                 const mode = plan?.modes.get(folder);
+                const backfill = quickable && !located;
                 const usable = Date.parse(point.full.since) <= since.getTime() && startedAt - Date.parse(point.full.at) < FULL_PASS_MS;
                 // A budgeted run goes on with an unfinished pass however small its window is: listing it whole again would
                 // ask for the same newest messages every run.
                 const unfinished = plan !== undefined && searched.uids.some((uid) => !within(point.done, Number(uid)));
                 if (mode === "resumed" ||
-                    (mode === undefined && usable && (searched.uids.length >= resumeFrom || unfinished))) {
+                    (mode === undefined && usable && !backfill && (searched.uids.length >= resumeFrom || unfinished))) {
                     resumed.add(folder);
                     wanted = { ...searched, uids: searched.uids.filter((uid) => !within(point.done, Number(uid))) };
                 }
@@ -332,6 +317,22 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
         current.add(item.folder ?? folders[0]);
         foundFolders.set(item.messageId, current);
     }
+    const listingWhole = !capped && (plan === undefined || (plan.stopped === null && plan.failed.size === 0 && plan.only === undefined));
+    /** Where each message was seen this run, saved with it so a message saved is a message located. */
+    const locationsOf = new Map();
+    for (const item of collected) {
+        const folder = item.folder ?? folders[0];
+        const validity = searches.get(folder)?.validity;
+        if (!locatable.has(folder) || validity === undefined)
+            continue;
+        const where = locationsOf.get(item.messageId) ?? [];
+        where.push({ folder, uidvalidity: validity, uid: Number(item.uid) });
+        locationsOf.set(item.messageId, where);
+    }
+    const locationOf = (item) => {
+        const locations = locationsOf.get(item.messageId);
+        return locations === undefined ? {} : { locations };
+    };
     const mailbox = (folder) => ({ externalId: folder, name: folder, kind: null });
     const label = (name) => ({ externalId: `label:${name}`, name, kind: "label" });
     const labelsOf = new Map(collected.flatMap((item) => (item.labels === undefined ? [] : [[item.messageId, item]])));
@@ -346,8 +347,23 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
         skipped = "an mbox file is a snapshot; mail missing from it is not proven deleted";
     else if (plan !== undefined && (capped || plan.tried.size > plan.asked || plan.only !== undefined))
         skipped = "this run read the window in parts; a run that lists all of it at once checks deletions";
-    else if (quickRan.size > 0)
-        skipped = "this run listed only mail not read before, by UID; --rescan lists every message and checks deletions";
+    else if (quickRan.size > 0) {
+        const proof = await proveByLocation(store, accountId, {
+            folders: new Map([...quickRan].map((folder) => [
+                folder,
+                { searched: searches.get(folder), rows: folderMail.get(folder) ?? [] },
+            ])),
+            windowFrom: windowStart.getTime(),
+            present,
+            now: startedAt,
+        }, tooManyGone);
+        deleted = proof.deleted;
+        skipped =
+            proof.skipped ??
+                (quickRan.size < folders.length
+                    ? "folders this run did not list by UID are not checked for deletions; --rescan lists every message and checks them"
+                    : undefined);
+    }
     else if (resumed.size > 0)
         skipped =
             "this run listed only mail not read before; a full listing, at least daily or with --rescan, checks deletions";
@@ -481,6 +497,8 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
                     ? undefined
                     : { since: prior.window.since, total: prior.window.total, at: prior.updatedAt });
             const widestEver = priorWidest === undefined || since.getTime() <= Date.parse(priorWidest.since) ? thisRun : priorWidest;
+            const located = locatable.has(folder) && (quickRan.has(folder) || (!resumed.has(folder) && listingWhole));
+            const locatedFrom = before?.located === undefined ? since.toISOString() : before.located;
             const point = {
                 validity: searched.validity,
                 done: merged,
@@ -494,12 +512,22 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
                     }
                     : before.window,
                 widest: widestEver,
+                ...(located
+                    ? { located: Date.parse(locatedFrom) < since.getTime() ? locatedFrom : since.toISOString() }
+                    : before?.located === undefined
+                        ? {}
+                        : { located: before.located }),
                 last: { ...thisRun, done: searched.uids.filter((uid) => within(merged, Number(uid))).length },
                 updatedAt: new Date(now()).toISOString(),
             };
             await store.sync.setSyncState(key, resumeKey(folder), JSON.stringify(point));
         }
     };
+    // Mail already stored is located before its UID is marked done, so a done UID always has its place.
+    for (const folder of locatable) {
+        const searched = searches.get(folder);
+        await store.mail.setLocations(accountId, folder, searched.validity, uidsOf.get(folder) ?? new Map());
+    }
     let savedAt = now();
     const saveNow = async (force = false) => {
         if (!force && now() - savedAt < SAVE_EVERY_MS)
@@ -534,6 +562,7 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
                     size: item.size ?? null,
                     outgoing: headers?.from === key.account,
                     mailboxes: [...inMailboxes(item), envelopeBox(rule.match)],
+                    ...locationOf(item),
                     ...flagsOf(item),
                     mailRuleId: rule.id ?? null,
                     metadata: {
@@ -620,6 +649,7 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
                         ...(Object.keys(header).length ? { headers: header } : {}),
                         outgoing: mail.from === null ? null : mail.from.email === key.account,
                         mailboxes: inMailboxes(item),
+                        ...locationOf(item),
                         ...flagsOf(item),
                         ...(upgraded.has(item.messageId) ? { metadata: {} } : {}),
                         mailRuleId: decide(rules, factsFor(item))?.id ?? null,
@@ -647,9 +677,17 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
     }
     // Read and starred change at the source after import; mail saved in this run already holds them.
     if (!snapshot)
-        await store.mail.setFlags(accountId, new Map([...labelsOf].flatMap(([id, { seen, flagged }]) => seen === undefined && flagged === undefined
-            ? []
-            : [[id, { ...(seen === undefined ? {} : { read: seen }), ...(flagged === undefined ? {} : { flagged }) }]])), startedAt);
+        await store.mail.setFlags(accountId, new Map([
+            ...refreshed,
+            ...[...labelsOf].flatMap(([id, { seen, flagged }]) => seen === undefined && flagged === undefined
+                ? []
+                : [
+                    [
+                        id,
+                        { ...(seen === undefined ? {} : { read: seen }), ...(flagged === undefined ? {} : { flagged }) },
+                    ],
+                ]),
+        ]), startedAt);
     const incomplete = capped || (plan !== undefined && (plan.stopped !== null || plan.failed.size > 0));
     const complete = !full && !incomplete && unreadable.length === 0 && (plan?.unreadable.size ?? 0) === 0;
     const bulk = new Map(listed.filter((item) => item.headers?.bulk).map((item) => [item.messageId, [bulkBox.externalId]]));
@@ -690,108 +728,3 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
         ...(tagging.skipped.length ? { tagsSkipped: tagging.skipped } : {}),
     };
 };
-/** Failures a newer install mends: the reader was missing, not the file at fault, so reading again can succeed. */
-export const RETRYABLE_ATTACHMENTS = ["engine-missing"];
-/**
- * Reads again, by Gmail id, only the stored emails with an attachment a missing reader left unread: memo keeps no
- * attachment file, so the email comes from the source again. Nothing is listed, so no deletion is proved and no
- * resume point moves; the email keeps its stored mailboxes, flags and rule.
- */
-const retryUnread = async ({ store, source, account, since, max, maxBodyChars, maxAttachmentChars, now, loadEngine, index, }) => {
-    const startedAt = now();
-    const key = { provider: "email", account: account.address.toLowerCase() };
-    const find = source.find;
-    if (find === undefined)
-        throw new CliError("validation_error", "--retry-attachments reads mail again by its Gmail id; an mbox file or a plain IMAP account has none");
-    const accountId = await store.accounts.saveAccount(key, { name: account.address });
-    const wanted = (await store.mail.emailRefs({ accountId, attachmentErrors: RETRYABLE_ATTACHMENTS })).map(({ externalId }) => externalId);
-    const chosen = wanted.slice(0, max);
-    const found = new Map();
-    for (const folder of await mailFolders(account, source)) {
-        const left = chosen.filter((id) => !found.has(id));
-        if (left.length === 0)
-            break;
-        for (const item of await find(left, folder))
-            if (!found.has(item.messageId))
-                found.set(item.messageId, item);
-    }
-    const items = [...found.values()];
-    const attachments = [];
-    const unreadable = [];
-    let emails = 0;
-    let extracted = 0;
-    let stillFailed = 0;
-    for (let start = 0; start < items.length; start += FETCH_CHUNK) {
-        const chunk = items.slice(start, start + FETCH_CHUNK);
-        const raws = await source.fetchRaw(chunk);
-        for (const [index, item] of chunk.entries()) {
-            const raw = raws[index];
-            const stored = await store.mail.email(accountId, item.messageId);
-            if (stored === null)
-                continue;
-            let mail;
-            try {
-                if (raw === undefined || raw instanceof Error)
-                    throw raw ?? new Error("not read");
-                mail = await parseMail(raw, maxBodyChars);
-            }
-            catch (error) {
-                unreadable.push({ message: item.messageId, reason: error instanceof Error ? error.message : String(error) });
-                continue;
-            }
-            const before = new Map(stored.attachments.map(({ position, extractionError }) => [position, extractionError]));
-            const files = await readAttachments(mail, item.messageId, { loadEngine, maxAttachmentChars, at: startedAt }, attachments);
-            for (const file of files)
-                if (RETRYABLE_ATTACHMENTS.includes(before.get(file.position) ?? ""))
-                    if (file.extraction === "text")
-                        extracted++;
-                    else
-                        stillFailed++;
-            const header = mailHeadersOf(raw);
-            await store.mail.saveThread({
-                accountId,
-                externalId: item.threadId,
-                now: startedAt,
-                emails: [
-                    {
-                        externalId: item.messageId,
-                        subject: mail.subject,
-                        from: mail.from === null ? null : addressOf(mail.from),
-                        to: mail.to.map(addressOf),
-                        cc: mail.cc.map(addressOf),
-                        bcc: mail.bcc.map(addressOf),
-                        sentAt: null,
-                        receivedAt: Date.parse(item.receivedAt),
-                        bodyText: mail.text || null,
-                        size: item.size ?? stored.size,
-                        ...(Object.keys(header).length ? { headers: header } : {}),
-                        outgoing: mail.from === null ? null : mail.from.email === key.account,
-                        mailboxes: stored.mailboxes.map(({ externalId, name, kind }) => ({ externalId, name, kind })),
-                        ...(stored.read === null ? {} : { read: stored.read }),
-                        ...(stored.flagged === null ? {} : { flagged: stored.flagged }),
-                        metadata: stored.metadata,
-                        mailRuleId: stored.mailRuleId,
-                        attachments: files,
-                    },
-                ],
-            });
-            emails++;
-        }
-    }
-    const indexed = await index?.(store, key);
-    return {
-        account: key.account,
-        since: since.toISOString(),
-        listed: items.length,
-        saved: 0,
-        alreadyStored: 0,
-        skipped: 0,
-        envelopes: 0,
-        deleted: 0,
-        complete: wanted.length <= max && unreadable.length === 0,
-        ...(indexed === undefined ? {} : { indexed }),
-        retried: { emails, extracted, stillFailed, missing: chosen.length - items.length },
-        ...(attachments.length ? { attachments } : {}),
-        ...(unreadable.length ? { unreadable } : {}),
-    };
-};

package/dist/mail/testing/fake.d.ts

@@ -26,6 +26,8 @@ export interface FakeMail {
     flagged?: boolean;
     /** `trash` puts it in the trash folder; otherwise it is in All Mail, which INBOX also shows. */
     folder?: "trash";
+    /** Folders that do not hold it although its `folder` says they do, as a server's delete from one folder leaves it. */
+    absentFrom?: string[];
 }
 /** The mailbox a fake source reads live, so a test changes it between syncs; the switches make the source fail. */
 export interface FakeMailbox {

package/dist/mail/testing/fake.js

@@ -54,7 +54,7 @@ export const fakeMailSource = (mailbox) => {
     const inFolder = (folder) => {
         if (![all(), trash(), "INBOX"].includes(folder))
             throw failure(`Gmail has no such mailbox: ${folder}`);
-        return mailbox.messages.filter((m) => (folder === trash() ? m.folder === "trash" : m.folder !== "trash"));
+        return mailbox.messages.filter((m) => (folder === trash() ? m.folder === "trash" : m.folder !== "trash") && !m.absentFrom?.includes(folder));
     };
     const search = async (since, folder) => {
         if (mailbox.searchFails)

package/dist/store/index.d.ts

@@ -29,7 +29,7 @@ export type { BotUpdate, BotUpdateStore } from "./sqlite/bot-updates.js";
 export type { CatalogActor, CatalogEntry, CatalogInput, CatalogQuery, CatalogStore } from "./sqlite/catalog.js";
 export { CATALOG_QUERY } from "./sqlite/catalog.js";
 export type { ChatMetadata } from "./sqlite/chats.js";
-export type { Email, EmailAddress, EmailAttachment, EmailFacts, EmailFilter, EmailInput, EmailInThread, EmailRecipient, EmailRef, EmailThread, EmailVectorHit, Mailbox, MailboxInput, MailFilter, MailStore, SenderScore, ThreadDetails, ThreadSave, } from "./sqlite/emails.js";
+export type { Email, EmailAddress, EmailAttachment, EmailFacts, EmailFilter, EmailInput, EmailInThread, EmailLocationInput, EmailRecipient, EmailRef, EmailThread, EmailVectorHit, FolderEmail, Mailbox, MailboxInput, MailFilter, MailStore, SenderScore, ThreadDetails, ThreadSave, } from "./sqlite/emails.js";
 export { EVENT_FIELDS, type EventChunkToEmbed, type EventHit, type EventQuery, type EventSearch, type NearestEvent, } from "./sqlite/event-search.js";
 export type { Fact, FactInput, FactsStore } from "./sqlite/facts.js";
 export type { FileListQuery, FilesStore, StoredFile } from "./sqlite/files.js";

package/dist/store/sqlite/accounts.js

@@ -92,6 +92,7 @@ export const purgeAccount = ({ orm, database }, accountKey) => {
     orm.delete(messages).where(eq(messages.accountId, accountKey)).run();
     orm.delete(chats).where(eq(chats.accountId, accountKey)).run();
     orm.delete(syncCursors).where(eq(syncCursors.accountId, accountKey)).run();
+    orm.run(sql `DELETE FROM email_locations WHERE mailbox_id IN (SELECT id FROM mailboxes WHERE account_id = ${accountKey})`);
     // Deleting an email clears its recipients, mailboxes, words, attachments and tags (trigger email_bd).
     orm.run(sql `DELETE FROM emails WHERE account_id = ${accountKey}`);
     orm.run(sql `DELETE FROM taggings WHERE taggable_type = 'email_thread' AND taggable_id IN (SELECT id FROM email_threads WHERE account_id = ${accountKey})`);

package/dist/store/sqlite/email-locations.d.ts

@@ -0,0 +1,24 @@
+import type { CacheDatabase } from "../driver.js";
+/** Where an IMAP server holds an email: `folder` is the mailbox's external id. */
+export interface EmailLocationInput {
+    folder: string;
+    uidvalidity: string;
+    uid: number;
+}
+/** One email of a folder as a sync sees it there. */
+export interface FolderEmail {
+    uid: number;
+    externalId: string;
+    receivedAt: number | null;
+    read: boolean | null;
+    flagged: boolean | null;
+    /** Its locations in every folder, and the folders it is a member of: an email is gone only when all are lost. */
+    locations: number;
+    folders: number;
+}
+export declare const locate: (database: CacheDatabase, accountId: number, emailId: number, where: EmailLocationInput) => {
+    changes: number;
+};
+export declare const folderEmails: (database: CacheDatabase, accountId: number, folder: string, uidvalidity: string) => FolderEmail[];
+export declare const setLocations: (database: CacheDatabase, accountId: number, folder: string, uidvalidity: string, found: ReadonlyMap<string, number[]>) => number;
+export declare const forgetLocations: (database: CacheDatabase, accountId: number, folder: string, uidvalidity: string, uids: number[]) => void;

package/dist/store/sqlite/email-locations.js

@@ -0,0 +1,56 @@
+export const locate = (database, accountId, emailId, where) => database
+    .prepare(`INSERT OR REPLACE INTO email_locations (mailbox_id, uidvalidity, uid, email_id)
+         SELECT id, ?, ?, ? FROM mailboxes WHERE account_id = ? AND external_id = ?`)
+    .run(where.uidvalidity, where.uid, emailId, accountId, where.folder);
+export const folderEmails = (database, accountId, folder, uidvalidity) => database
+    .prepare(`SELECT l.uid, e.external_id, coalesce(e.received_at, e.sent_at) AS at, e.read, e.flagged,
+         (SELECT count(*) FROM email_locations x WHERE x.email_id = e.id) AS locations,
+         (SELECT count(*) FROM email_mailboxes m JOIN mailboxes b ON b.id = m.mailbox_id
+           WHERE m.email_id = e.id AND b.kind IS NULL) AS folders
+       FROM email_locations l JOIN emails e ON e.id = l.email_id
+       WHERE l.mailbox_id = (SELECT id FROM mailboxes WHERE account_id = ? AND external_id = ?)
+         AND l.uidvalidity = ? AND e.deleted_at IS NULL
+       ORDER BY l.uid`)
+    .all(accountId, folder, uidvalidity)
+    .map((row) => ({
+    uid: Number(row.uid),
+    externalId: String(row.external_id),
+    receivedAt: row.at === null ? null : Number(row.at),
+    read: row.read === null ? null : Boolean(row.read),
+    flagged: row.flagged === null ? null : Boolean(row.flagged),
+    locations: Number(row.locations),
+    folders: Number(row.folders),
+}));
+export const setLocations = (database, accountId, folder, uidvalidity, found) => {
+    database
+        .prepare(`DELETE FROM email_locations WHERE uidvalidity <> ?
+         AND mailbox_id = (SELECT id FROM mailboxes WHERE account_id = ? AND external_id = ?)`)
+        .run(uidvalidity, accountId, folder);
+    const lookup = database.prepare("SELECT id FROM emails WHERE account_id = ? AND external_id = ?");
+    let count = 0;
+    for (const [externalId, uids] of found) {
+        const row = lookup.get(accountId, externalId);
+        if (!row)
+            continue;
+        for (const uid of uids)
+            locate(database, accountId, Number(row.id), { folder, uidvalidity, uid });
+        count++;
+    }
+    return count;
+};
+export const forgetLocations = (database, accountId, folder, uidvalidity, uids) => {
+    const box = database
+        .prepare("SELECT id FROM mailboxes WHERE account_id = ? AND external_id = ?")
+        .get(accountId, folder);
+    if (!box)
+        return;
+    const emails = database
+        .prepare(`DELETE FROM email_locations WHERE mailbox_id = ? AND uidvalidity = ?
+         AND uid IN (SELECT value FROM json_each(?)) RETURNING email_id`)
+        .all(Number(box.id), uidvalidity, JSON.stringify(uids))
+        .map((row) => Number(row.email_id));
+    const leave = database.prepare(`DELETE FROM email_mailboxes WHERE email_id = ? AND mailbox_id = ?
+       AND NOT EXISTS (SELECT 1 FROM email_locations WHERE email_id = ? AND mailbox_id = ?)`);
+    for (const emailId of new Set(emails))
+        leave.run(emailId, Number(box.id), emailId, Number(box.id));
+};

package/dist/store/sqlite/email-state.d.ts

@@ -6,3 +6,5 @@ export declare const setEmailFlags: (database: CacheDatabase, accountId: number,
     read?: boolean;
     flagged?: boolean;
 }>, now: number) => number;
+export declare const markEmailsDeleted: (database: CacheDatabase, accountId: number, externalIds: string[], now: number) => number;
+export declare const markEmailsRestored: (database: CacheDatabase, accountId: number, externalIds: string[], now: number) => number;

package/dist/store/sqlite/email-state.js

@@ -1,4 +1,6 @@
+import { dropAttachmentChunks } from "./email-index.js";
 import { flag } from "./events.js";
+import { dropUnusedVectors } from "./vector-index.js";
 export const recountEmailThread = (database, threadId, now) => database
     .prepare(`UPDATE email_threads SET
          emails_count = (SELECT count(*) FROM emails WHERE email_thread_id = ? AND deleted_at IS NULL),
@@ -23,3 +25,29 @@ export const setEmailFlags = (database, accountId, flags, now) => {
     }
     return changed;
 };
+export const markEmailsDeleted = (database, accountId, externalIds, now) => {
+    const mark = database.prepare(`UPDATE emails SET deleted_at = ?, body_text = NULL, body_html = NULL, snippet = NULL, updated_at = ?
+       WHERE account_id = ? AND external_id = ? AND deleted_at IS NULL
+       RETURNING id, email_thread_id`);
+    const forget = database.prepare("UPDATE attachments SET text = NULL, normalized_text = NULL WHERE attachable_type = 'email' AND attachable_id = ?");
+    const unlocate = database.prepare("DELETE FROM email_locations WHERE email_id = ?");
+    const threads = externalIds.flatMap((id) => mark.all(now, now, accountId, id).map((r) => {
+        const pieces = dropAttachmentChunks(database, Number(r.id));
+        forget.run(Number(r.id));
+        unlocate.run(Number(r.id));
+        dropUnusedVectors(database, pieces);
+        return Number(r.email_thread_id);
+    }));
+    for (const id of new Set(threads))
+        recountEmailThread(database, id, now);
+    return threads.length;
+};
+export const markEmailsRestored = (database, accountId, externalIds, now) => {
+    const unmark = database.prepare(`UPDATE emails SET deleted_at = NULL, updated_at = ?
+       WHERE account_id = ? AND external_id = ? AND deleted_at IS NOT NULL
+       RETURNING email_thread_id`);
+    const threads = externalIds.flatMap((id) => unmark.all(now, accountId, id).map((r) => Number(r.email_thread_id)));
+    for (const id of new Set(threads))
+        recountEmailThread(database, id, now);
+    return threads.length;
+};

package/dist/store/sqlite/emails.d.ts

@@ -3,9 +3,11 @@ import type { Attachment, Metadata } from "../../meetings/index.js";
 import type { Stemmer, Stemmers } from "../../search/stem.js";
 import type { CacheDatabase } from "../driver.js";
 import type { ConversationSummary, StoredHit } from "../types.js";
+import { type EmailLocationInput, type FolderEmail } from "./email-locations.js";
 import { type MailRulesStore } from "./mail-rules.js";
 import { type AttachmentInput } from "./meetings.js";
 import type { StoreContext } from "./open.js";
+export type { EmailLocationInput, FolderEmail };
 export interface EmailAddress {
     address: string;
     name: string | null;
@@ -43,6 +45,8 @@ export interface EmailInput {
     mailRuleId?: number | null;
     /** Added to the mailboxes the email is already in; one scan sees one folder, so none is taken away. */
     mailboxes?: MailboxInput[];
+    /** Added to the locations the email has; a UID already known in that folder now points here. */
+    locations?: EmailLocationInput[];
     /** Replaced when given; the text is word-indexed. */
     attachments?: AttachmentInput[];
 }
@@ -228,6 +232,15 @@ export interface MailStore {
         read?: boolean;
         flagged?: boolean;
     }>, now: number): Promise<number>;
+    /** The live emails whose location in `folder` has this UIDVALIDITY, oldest UID first. */
+    folderEmails(accountId: number, folder: string, uidvalidity: string): Promise<FolderEmail[]>;
+    /**
+     * Records where emails the store holds are in `folder`, by Message-ID; one it does not hold is left out. Locations
+     * of the folder under another UIDVALIDITY are dropped: its UIDs mean nothing now.
+     */
+    setLocations(accountId: number, folder: string, uidvalidity: string, found: ReadonlyMap<string, number[]>): Promise<number>;
+    /** Forgets that these UIDs of `folder` hold anything; an email left with no location there leaves the folder too. */
+    forgetLocations(accountId: number, folder: string, uidvalidity: string, uids: number[]): Promise<void>;
     /** Marks emails gone at the source and drops their text; saving one again brings it back. Answers how many changed. */
     markDeleted(accountId: number, externalIds: string[], now: number): Promise<number>;
     /**

package/dist/store/sqlite/emails.js

@@ -2,7 +2,8 @@ import { formatLocator } from "../../domain/locator.js";
 import { formatReference } from "../../domain/references.js";
 import { fold, normalize } from "../normalize.js";
 import { chunkAttachments, drainEmailIndex, dropAttachmentChunks, MAIL_PIECES, queueThreads, threadPieces, titleOf, } from "./email-index.js";
-import { recountEmailThread as recount, setEmailFlags } from "./email-state.js";
+import { folderEmails, forgetLocations, locate, setLocations, } from "./email-locations.js";
+import { markEmailsDeleted, markEmailsRestored, recountEmailThread as recount, setEmailFlags } from "./email-state.js";
 import { bool, flag, fromJson, int, page, str, toJson } from "./events.js";
 import { folded } from "./folded.js";
 import { ensurePerson } from "./identities.js";
@@ -228,6 +229,8 @@ const saveEmail = (database, orm, accountId, threadId, mail, now) => {
     }
     for (const box of mail.mailboxes ?? [])
         join(database, emailId, mailboxPk(database, accountId, box, now), now);
+    for (const where of mail.locations ?? [])
+        locate(database, accountId, emailId, where);
     if (mail.attachments !== undefined) {
         const before = dropAttachmentChunks(database, emailId);
         const positions = mail.attachments.map(({ position }) => position);
@@ -545,34 +548,11 @@ export const mailStoreOver = ({ database, orm }, stemmerFor = stemmerCache()) =>
             return setEmailFlags(database, accountId, flags, now);
         });
     },
-    async markDeleted(accountId, externalIds, now) {
-        return write(database, () => {
-            const mark = database.prepare(`UPDATE emails SET deleted_at = ?, body_text = NULL, body_html = NULL, snippet = NULL, updated_at = ?
-           WHERE account_id = ? AND external_id = ? AND deleted_at IS NULL
-           RETURNING id, email_thread_id`);
-            const forget = database.prepare("UPDATE attachments SET text = NULL, normalized_text = NULL WHERE attachable_type = 'email' AND attachable_id = ?");
-            const threads = externalIds.flatMap((id) => mark.all(now, now, accountId, id).map((r) => {
-                const pieces = dropAttachmentChunks(database, Number(r.id));
-                forget.run(Number(r.id));
-                dropUnusedVectors(database, pieces);
-                return Number(r.email_thread_id);
-            }));
-            for (const id of new Set(threads))
-                recount(database, id, now);
-            return threads.length;
-        });
-    },
-    async markRestored(accountId, externalIds, now) {
-        return write(database, () => {
-            const unmark = database.prepare(`UPDATE emails SET deleted_at = NULL, updated_at = ?
-           WHERE account_id = ? AND external_id = ? AND deleted_at IS NOT NULL
-           RETURNING email_thread_id`);
-            const threads = externalIds.flatMap((id) => unmark.all(now, accountId, id).map((r) => Number(r.email_thread_id)));
-            for (const id of new Set(threads))
-                recount(database, id, now);
-            return threads.length;
-        });
-    },
+    folderEmails: async (accountId, folder, uidvalidity) => folderEmails(database, accountId, folder, uidvalidity),
+    setLocations: async (accountId, folder, uidvalidity, found) => write(database, () => setLocations(database, accountId, folder, uidvalidity, found)),
+    forgetLocations: async (accountId, folder, uidvalidity, uids) => write(database, () => forgetLocations(database, accountId, folder, uidvalidity, uids)),
+    markDeleted: async (accountId, externalIds, now) => write(database, () => markEmailsDeleted(database, accountId, externalIds, now)),
+    markRestored: async (accountId, externalIds, now) => write(database, () => markEmailsRestored(database, accountId, externalIds, now)),
     async chunksToEmbed(model, { after, limit }) {
         drainEmailIndex(database, stemmerFor);
         const file = database.prepare("SELECT text FROM attachments WHERE id = ?");

package/dist/store/sqlite/manifest.js

@@ -52,6 +52,7 @@ export const MANIFEST = [
     { name: "20261029181000_version-37-contact-mention-triggers", version: 37, minCompatible: 34 },
     { name: "20261029224106_version-38-folded-names", version: 38, minCompatible: 34 },
     { name: "20261029224107_version-38-folded-triggers", version: 38, minCompatible: 34 },
+    { name: "20261030114008_version-39-email-locations", version: 39, minCompatible: 34 },
 ];
 export const generatedMigrations = (manifest = MANIFEST, generated = GENERATED) => {
     const migrations = [];

package/dist/store/sqlite/migrations.generated.js

@@ -830,5 +830,12 @@ export const GENERATED = [
             "CREATE TRIGGER email_threads_subject_folded_au AFTER UPDATE OF subject ON email_threads\n  WHEN new.subject IS NOT old.subject AND new.subject_folded IS old.subject_folded BEGIN\n  UPDATE email_threads SET subject_folded = NULL WHERE id = new.id;\nEND;",
             "CREATE TRIGGER mailboxes_name_folded_au AFTER UPDATE OF name ON mailboxes\n  WHEN new.name IS NOT old.name AND new.name_folded IS old.name_folded BEGIN\n  UPDATE mailboxes SET name_folded = NULL WHERE id = new.id;\nEND;"
         ]
+    },
+    {
+        "name": "20261030114008_version-39-email-locations",
+        "statements": [
+            "CREATE TABLE `email_locations` (\n\t`mailbox_id` integer NOT NULL,\n\t`uidvalidity` text NOT NULL,\n\t`uid` integer NOT NULL,\n\t`email_id` integer NOT NULL,\n\tCONSTRAINT `email_locations_pk` PRIMARY KEY(`mailbox_id`, `uidvalidity`, `uid`),\n\tCONSTRAINT `fk_email_locations_mailbox_id_mailboxes_id_fk` FOREIGN KEY (`mailbox_id`) REFERENCES `mailboxes`(`id`),\n\tCONSTRAINT `fk_email_locations_email_id_emails_id_fk` FOREIGN KEY (`email_id`) REFERENCES `emails`(`id`)\n);",
+            "CREATE INDEX `email_locations_by_email_id` ON `email_locations` (`email_id`);"
+        ]
     }
 ];

package/dist/store/sqlite/schema.d.ts

@@ -6172,6 +6172,78 @@ export declare const emailMailboxes: import("drizzle-orm/sqlite-core").SQLiteTab
     dialect: 'sqlite';
     isAlias: false;
 }>;
+export declare const emailLocations: import("drizzle-orm/sqlite-core").SQLiteTableWithColumns<{
+    name: "email_locations";
+    schema: undefined;
+    columns: {
+        mailboxId: import("drizzle-orm/sqlite-core").SQLiteColumn<{
+            name: string;
+            tableName: "email_locations";
+            dataType: "number int53";
+            data: number;
+            driverParam: number;
+            notNull: true;
+            hasDefault: false;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: undefined;
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}>;
+        uidvalidity: import("drizzle-orm/sqlite-core").SQLiteColumn<{
+            name: string;
+            tableName: "email_locations";
+            dataType: "string";
+            data: string;
+            driverParam: string;
+            notNull: true;
+            hasDefault: false;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: [string, ...string[]];
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}>;
+        uid: import("drizzle-orm/sqlite-core").SQLiteColumn<{
+            name: string;
+            tableName: "email_locations";
+            dataType: "number int53";
+            data: number;
+            driverParam: number;
+            notNull: true;
+            hasDefault: false;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: undefined;
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}>;
+        emailId: import("drizzle-orm/sqlite-core").SQLiteColumn<{
+            name: string;
+            tableName: "email_locations";
+            dataType: "number int53";
+            data: number;
+            driverParam: number;
+            notNull: true;
+            hasDefault: false;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: undefined;
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}>;
+    };
+    dialect: 'sqlite';
+    isAlias: false;
+}>;
 export declare const emailIndexPending: import("drizzle-orm/sqlite-core").SQLiteTableWithColumns<{
     name: "email_index_pending";
     schema: undefined;

package/dist/store/sqlite/schema.js

@@ -695,6 +695,19 @@ export const emailMailboxes = sqliteTable("email_mailboxes", {
     primaryKey({ columns: [table.emailId, table.mailboxId] }),
     index("email_mailboxes_by_mailbox_id").on(table.mailboxId),
 ]);
+export const emailLocations = sqliteTable("email_locations", {
+    mailboxId: integer("mailbox_id")
+        .notNull()
+        .references(() => mailboxes.id),
+    uidvalidity: text("uidvalidity").notNull(),
+    uid: integer("uid").notNull(),
+    emailId: integer("email_id")
+        .notNull()
+        .references(() => emails.id),
+}, (table) => [
+    primaryKey({ columns: [table.mailboxId, table.uidvalidity, table.uid] }),
+    index("email_locations_by_email_id").on(table.emailId),
+]);
 export const emailIndexPending = sqliteTable("email_index_pending", {
     id: integer("id").notNull(),
     indexableType: text("indexable_type").notNull(),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wirecat/cli-core",
-  "version": "0.487.0",
+  "version": "0.488.0",
   "description": "The messenger-neutral half of a messaging CLI: domain model, message locators, rendering, name resolution, the SQLite seam, the command skeleton and shared read commands, the send guard, run records and the message store",
   "license": "Apache-2.0",
   "author": "Viacheslav Ptsarev",