@wirecat/cli-core 0.486.0 → 0.488.0

package/dist/mail/sync.js

@@ -1,61 +1,28 @@
 import { CliError } from "@wirecat/cli-base";
-import { extractText, importEngine } from "../attachments/extract.js";
+import { importEngine } from "../attachments/extract.js";
+import { addressOf, FETCH_CHUNK, MAX_ATTACHMENT_CHARS, readAttachments } from "./attachments.js";
 import { BULK, bulkBox } from "./bulk.js";
 import { factsOf } from "./facts.js";
 import { GMAIL_ALL_MAIL, mailFolders } from "./folders.js";
 import { dropMissing, tooManyGone } from "./gone.js";
 import { mailHeadersOf } from "./headers.js";
+import { flagsToRefresh, proveByLocation } from "./locations.js";
 import { MAX_BODY_CHARS, parseMail } from "./parse.js";
+import { FETCH_SLICE, keyOf, readQuick } from "./quick.js";
 import { FULL_PASS_MS, MBOX_RESUME, mergeRanges, parsePoint, rangesOf, resumeKey, rulesKey, within, } from "./resume.js";
+import { retryUnread } from "./retry.js";
 import { ruleTagger, settleRuleTags } from "./rule-tags.js";
 import { decide, ENVELOPE, envelopeBox, envelopeBoxOf, tagsOf } from "./rules.js";
 const DAY = 86_400_000;
 /** IMAP's SINCE counts whole days in the server's zone, so the window's first days are not trusted. */
 const EDGE = 2 * DAY;
-export const MAX_ATTACHMENT_CHARS = 200_000;
-/** One chunk of raw mail is held in memory at a time. */
-const FETCH_CHUNK = 50;
 /** How often a long run saves where it is, so a kill costs at most this much listing again. */
 const SAVE_EVERY_MS = 30_000;
 /** About a minute of listing; the 30-day window of the timer stays below it. */
 const RESUME_FROM = 5_000;
-const addressOf = ({ email, name }) => ({ address: email, name });
-/** A parsed email's attachments as the store keeps them, each with its text or why it has none. */
-const readAttachments = async (mail, messageId, { loadEngine, maxAttachmentChars, at }, log) => {
-    const files = [];
-    for (const [position, part] of mail.attachments.entries()) {
-        const extraction = part.bytes === null
-            ? { status: "unavailable" }
-            : await extractText(part.bytes, { kind: "file", name: part.name, mime: part.mime, path: part.name }, loadEngine);
-        log.push({ message: messageId, part: part.id, status: extraction.status });
-        const extracted = extraction.status === "extracted" ? extraction : undefined;
-        files.push({
-            position,
-            kind: "file",
-            mime: part.mime,
-            name: part.name,
-            title: null,
-            url: null,
-            size: part.bytes === null ? null : part.bytes.byteLength,
-            width: null,
-            height: null,
-            duration: null,
-            providerRef: { message: messageId, part: part.id },
-            localPath: null,
-            text: extracted ? extracted.text.slice(0, maxAttachmentChars) : null,
-            normalizedText: null,
-            extraction: extracted ? "text" : "failed",
-            extractor: extracted ? extracted.extractor : null,
-            extractionError: extracted ? null : extraction.status,
-            contentSha256: null,
-            extractedAt: at,
-        });
-    }
-    return files;
-};
+export { MAX_ATTACHMENT_CHARS } from "./attachments.js";
+export { RETRYABLE_ATTACHMENTS } from "./retry.js";
 const LIST_SLICE = 200;
-const FETCH_SLICE = 50;
-const keyOf = (folder, uid) => `${folder}\0${uid}`;
 export const syncMail = async (source, store, options) => {
     const { budget, index } = options;
     if (budget === undefined || options.retryAttachments || options.account.mode === "mbox")
@@ -68,6 +35,8 @@ export const syncMail = async (source, store, options) => {
         modes: new Map(),
         unreadable: new Map(),
         unlisted: new Set(),
+        quick: new Map(),
+        quickFolders: new Set(),
         stopped: null,
         remaining: 0,
         asked: 0,
@@ -100,6 +69,7 @@ export const syncMail = async (source, store, options) => {
     // A message the server failed on once is often fine a minute later; only one more try, then the next run.
     if (plan.stopped === null && plan.failed.size > 0) {
         plan.only = new Set(plan.failed);
+        plan.quick.clear();
         plan.failed.clear();
         plan.tried.clear();
         await goOn();
@@ -188,6 +158,22 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
             mboxPoint = point;
     }
     let capped = false;
+    const unreadable = [];
+    /** Raw mail read to learn its id, kept for the save so it is not read twice. */
+    const rawOf = new Map();
+    const quickRan = new Set();
+    /** Folders whose server gives no cheap id: their mail's UIDs are kept, so a UID listing can stand in for headers. */
+    const locatable = new Set();
+    /** A quick folder's stored mail as it was before this run changed anything, with the UID of each. */
+    const folderMail = new Map();
+    const refreshed = new Map();
+    let holding;
+    const holdsMail = async (folder) => {
+        holding ??= new Set((await store.mail.emailRefs({ accountId })).flatMap(({ mailboxes }) => mailboxes
+            .filter(({ kind }) => kind !== "label" && kind !== ENVELOPE && kind !== BULK)
+            .map((box) => box.externalId)));
+        return holding.has(folder);
+    };
     for (const folder of folders) {
         if (mboxPoint !== undefined)
             break;
@@ -212,15 +198,59 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
             if (kept !== undefined)
                 storedPoints.set(folder, kept);
             const point = fresh ? undefined : kept;
+            const sameRules = point !== undefined && point.validity === searched.validity && point.rules === ruleSet;
+            // Without a cheap id the mail's identity is in its bytes, so a folder is listed quickly only when no stored mail
+            // can be read twice: its done UIDs are known, or the store holds nothing from it. Otherwise (a new UIDVALIDITY,
+            // other rules, --rescan) the full header listing recognises what is stored without downloading it.
+            const quickable = !snapshot &&
+                searched.quick === true &&
+                searched.validity !== undefined &&
+                source.quick !== undefined &&
+                source.identify !== undefined;
+            if (quickable)
+                locatable.add(folder);
+            // The UID listing proves deletions only for mail whose UID is stored; a folder stored before locations existed
+            // gets them from one header listing.
+            const located = point?.located !== undefined && Date.parse(point.located) <= since.getTime();
+            const quickReady = async () => (sameRules && located) || !(await holdsMail(folder));
+            const inQuick = quickable &&
+                (plan === undefined
+                    ? await quickReady()
+                    : plan.quickFolders.has(folder) || (!plan.modes.has(folder) && (await quickReady())));
+            if (inQuick) {
+                plan?.quickFolders.add(folder);
+                plan?.modes.set(folder, "resumed");
+                resumed.add(folder);
+                quickRan.add(folder);
+                const done = point !== undefined && sameRules ? point.done : [];
+                if (point !== undefined && sameRules)
+                    earlier.set(folder, point);
+                const rows = await store.mail.folderEmails(accountId, folder, searched.validity);
+                folderMail.set(folder, rows);
+                const byUid = new Map(rows.map((row) => [String(row.uid), row]));
+                const round = await readQuick(source, since, folder, searched, done, max, plan, new Set(byUid.keys()));
+                for (const [id, flags] of flagsToRefresh(byUid, round.listing))
+                    refreshed.set(id, { ...refreshed.get(id), ...flags });
+                if (round.capped)
+                    capped = true;
+                for (const { item, raw } of round.read) {
+                    collected.push(item);
+                    rawOf.set(item, raw);
+                }
+                for (const bad of round.unreadable)
+                    unreadable.push(bad);
+                continue;
+            }
             if (point !== undefined && point.validity === searched.validity && point.rules === ruleSet) {
                 earlier.set(folder, point);
                 const mode = plan?.modes.get(folder);
+                const backfill = quickable && !located;
                 const usable = Date.parse(point.full.since) <= since.getTime() && startedAt - Date.parse(point.full.at) < FULL_PASS_MS;
                 // A budgeted run goes on with an unfinished pass however small its window is: listing it whole again would
                 // ask for the same newest messages every run.
                 const unfinished = plan !== undefined && searched.uids.some((uid) => !within(point.done, Number(uid)));
                 if (mode === "resumed" ||
-                    (mode === undefined && usable && (searched.uids.length >= resumeFrom || unfinished))) {
+                    (mode === undefined && usable && !backfill && (searched.uids.length >= resumeFrom || unfinished))) {
                     resumed.add(folder);
                     wanted = { ...searched, uids: searched.uids.filter((uid) => !within(point.done, Number(uid))) };
                 }
@@ -287,6 +317,22 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
         current.add(item.folder ?? folders[0]);
         foundFolders.set(item.messageId, current);
     }
+    const listingWhole = !capped && (plan === undefined || (plan.stopped === null && plan.failed.size === 0 && plan.only === undefined));
+    /** Where each message was seen this run, saved with it so a message saved is a message located. */
+    const locationsOf = new Map();
+    for (const item of collected) {
+        const folder = item.folder ?? folders[0];
+        const validity = searches.get(folder)?.validity;
+        if (!locatable.has(folder) || validity === undefined)
+            continue;
+        const where = locationsOf.get(item.messageId) ?? [];
+        where.push({ folder, uidvalidity: validity, uid: Number(item.uid) });
+        locationsOf.set(item.messageId, where);
+    }
+    const locationOf = (item) => {
+        const locations = locationsOf.get(item.messageId);
+        return locations === undefined ? {} : { locations };
+    };
     const mailbox = (folder) => ({ externalId: folder, name: folder, kind: null });
     const label = (name) => ({ externalId: `label:${name}`, name, kind: "label" });
     const labelsOf = new Map(collected.flatMap((item) => (item.labels === undefined ? [] : [[item.messageId, item]])));
@@ -301,6 +347,23 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
         skipped = "an mbox file is a snapshot; mail missing from it is not proven deleted";
     else if (plan !== undefined && (capped || plan.tried.size > plan.asked || plan.only !== undefined))
         skipped = "this run read the window in parts; a run that lists all of it at once checks deletions";
+    else if (quickRan.size > 0) {
+        const proof = await proveByLocation(store, accountId, {
+            folders: new Map([...quickRan].map((folder) => [
+                folder,
+                { searched: searches.get(folder), rows: folderMail.get(folder) ?? [] },
+            ])),
+            windowFrom: windowStart.getTime(),
+            present,
+            now: startedAt,
+        }, tooManyGone);
+        deleted = proof.deleted;
+        skipped =
+            proof.skipped ??
+                (quickRan.size < folders.length
+                    ? "folders this run did not list by UID are not checked for deletions; --rescan lists every message and checks them"
+                    : undefined);
+    }
     else if (resumed.size > 0)
         skipped =
             "this run listed only mail not read before; a full listing, at least daily or with --rescan, checks deletions";
@@ -339,7 +402,6 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
         await store.mail.setMailboxes(accountId, [...scanned.values()], new Map([...labelsOf].map(([id, { labels }]) => [id, (labels ?? []).map((name) => label(name).externalId)])), startedAt);
     }
     const attachments = [];
-    const unreadable = [];
     let saved = 0;
     let alreadyStored = 0;
     let ruledOut = 0;
@@ -435,6 +497,8 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
                     ? undefined
                     : { since: prior.window.since, total: prior.window.total, at: prior.updatedAt });
             const widestEver = priorWidest === undefined || since.getTime() <= Date.parse(priorWidest.since) ? thisRun : priorWidest;
+            const located = locatable.has(folder) && (quickRan.has(folder) || (!resumed.has(folder) && listingWhole));
+            const locatedFrom = before?.located === undefined ? since.toISOString() : before.located;
             const point = {
                 validity: searched.validity,
                 done: merged,
@@ -448,12 +512,22 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
                     }
                     : before.window,
                 widest: widestEver,
+                ...(located
+                    ? { located: Date.parse(locatedFrom) < since.getTime() ? locatedFrom : since.toISOString() }
+                    : before?.located === undefined
+                        ? {}
+                        : { located: before.located }),
                 last: { ...thisRun, done: searched.uids.filter((uid) => within(merged, Number(uid))).length },
                 updatedAt: new Date(now()).toISOString(),
             };
             await store.sync.setSyncState(key, resumeKey(folder), JSON.stringify(point));
         }
     };
+    // Mail already stored is located before its UID is marked done, so a done UID always has its place.
+    for (const folder of locatable) {
+        const searched = searches.get(folder);
+        await store.mail.setLocations(accountId, folder, searched.validity, uidsOf.get(folder) ?? new Map());
+    }
     let savedAt = now();
     const saveNow = async (force = false) => {
         if (!force && now() - savedAt < SAVE_EVERY_MS)
@@ -488,6 +562,7 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
                     size: item.size ?? null,
                     outgoing: headers?.from === key.account,
                     mailboxes: [...inMailboxes(item), envelopeBox(rule.match)],
+                    ...locationOf(item),
                     ...flagsOf(item),
                     mailRuleId: rule.id ?? null,
                     metadata: {
@@ -509,7 +584,9 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
     for (let start = 0; start < pending.length; start += fetchChunk) {
         await stopIfAborted();
         const chunk = pending.slice(start, start + fetchChunk);
-        if (plan !== undefined) {
+        // Mail read to learn its id is already in hand: a chunk of it costs no request.
+        const unread = chunk.filter((item) => !rawOf.has(item));
+        if (plan !== undefined && unread.length > 0) {
             const stop = plan.budget.take();
             if (stop) {
                 plan.stopped = stop;
@@ -519,7 +596,8 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
         }
         let raws;
         try {
-            raws = await source.fetchRaw(chunk);
+            const got = unread.length > 0 ? await source.fetchRaw(unread) : [];
+            raws = chunk.map((item) => rawOf.get(item) ?? got[unread.indexOf(item)]);
         }
         catch (error) {
             // What this run saved stays saved: the next one goes on from it.
@@ -571,6 +649,7 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
                         ...(Object.keys(header).length ? { headers: header } : {}),
                         outgoing: mail.from === null ? null : mail.from.email === key.account,
                         mailboxes: inMailboxes(item),
+                        ...locationOf(item),
                         ...flagsOf(item),
                         ...(upgraded.has(item.messageId) ? { metadata: {} } : {}),
                         mailRuleId: decide(rules, factsFor(item))?.id ?? null,
@@ -598,10 +677,18 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
     }
     // Read and starred change at the source after import; mail saved in this run already holds them.
     if (!snapshot)
-        await store.mail.setFlags(accountId, new Map([...labelsOf].flatMap(([id, { seen, flagged }]) => seen === undefined && flagged === undefined
-            ? []
-            : [[id, { ...(seen === undefined ? {} : { read: seen }), ...(flagged === undefined ? {} : { flagged }) }]])), startedAt);
-    const incomplete = plan !== undefined && (capped || plan.stopped !== null || plan.failed.size > 0);
+        await store.mail.setFlags(accountId, new Map([
+            ...refreshed,
+            ...[...labelsOf].flatMap(([id, { seen, flagged }]) => seen === undefined && flagged === undefined
+                ? []
+                : [
+                    [
+                        id,
+                        { ...(seen === undefined ? {} : { read: seen }), ...(flagged === undefined ? {} : { flagged }) },
+                    ],
+                ]),
+        ]), startedAt);
+    const incomplete = capped || (plan !== undefined && (plan.stopped !== null || plan.failed.size > 0));
     const complete = !full && !incomplete && unreadable.length === 0 && (plan?.unreadable.size ?? 0) === 0;
     const bulk = new Map(listed.filter((item) => item.headers?.bulk).map((item) => [item.messageId, [bulkBox.externalId]]));
     if (bulk.size)
@@ -641,108 +728,3 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
         ...(tagging.skipped.length ? { tagsSkipped: tagging.skipped } : {}),
     };
 };
-/** Failures a newer install mends: the reader was missing, not the file at fault, so reading again can succeed. */
-export const RETRYABLE_ATTACHMENTS = ["engine-missing"];
-/**
- * Reads again, by Gmail id, only the stored emails with an attachment a missing reader left unread: memo keeps no
- * attachment file, so the email comes from the source again. Nothing is listed, so no deletion is proved and no
- * resume point moves; the email keeps its stored mailboxes, flags and rule.
- */
-const retryUnread = async ({ store, source, account, since, max, maxBodyChars, maxAttachmentChars, now, loadEngine, index, }) => {
-    const startedAt = now();
-    const key = { provider: "email", account: account.address.toLowerCase() };
-    const find = source.find;
-    if (find === undefined)
-        throw new CliError("validation_error", "--retry-attachments reads mail again by its Gmail id; an mbox file or a plain IMAP account has none");
-    const accountId = await store.accounts.saveAccount(key, { name: account.address });
-    const wanted = (await store.mail.emailRefs({ accountId, attachmentErrors: RETRYABLE_ATTACHMENTS })).map(({ externalId }) => externalId);
-    const chosen = wanted.slice(0, max);
-    const found = new Map();
-    for (const folder of await mailFolders(account, source)) {
-        const left = chosen.filter((id) => !found.has(id));
-        if (left.length === 0)
-            break;
-        for (const item of await find(left, folder))
-            if (!found.has(item.messageId))
-                found.set(item.messageId, item);
-    }
-    const items = [...found.values()];
-    const attachments = [];
-    const unreadable = [];
-    let emails = 0;
-    let extracted = 0;
-    let stillFailed = 0;
-    for (let start = 0; start < items.length; start += FETCH_CHUNK) {
-        const chunk = items.slice(start, start + FETCH_CHUNK);
-        const raws = await source.fetchRaw(chunk);
-        for (const [index, item] of chunk.entries()) {
-            const raw = raws[index];
-            const stored = await store.mail.email(accountId, item.messageId);
-            if (stored === null)
-                continue;
-            let mail;
-            try {
-                if (raw === undefined || raw instanceof Error)
-                    throw raw ?? new Error("not read");
-                mail = await parseMail(raw, maxBodyChars);
-            }
-            catch (error) {
-                unreadable.push({ message: item.messageId, reason: error instanceof Error ? error.message : String(error) });
-                continue;
-            }
-            const before = new Map(stored.attachments.map(({ position, extractionError }) => [position, extractionError]));
-            const files = await readAttachments(mail, item.messageId, { loadEngine, maxAttachmentChars, at: startedAt }, attachments);
-            for (const file of files)
-                if (RETRYABLE_ATTACHMENTS.includes(before.get(file.position) ?? ""))
-                    if (file.extraction === "text")
-                        extracted++;
-                    else
-                        stillFailed++;
-            const header = mailHeadersOf(raw);
-            await store.mail.saveThread({
-                accountId,
-                externalId: item.threadId,
-                now: startedAt,
-                emails: [
-                    {
-                        externalId: item.messageId,
-                        subject: mail.subject,
-                        from: mail.from === null ? null : addressOf(mail.from),
-                        to: mail.to.map(addressOf),
-                        cc: mail.cc.map(addressOf),
-                        bcc: mail.bcc.map(addressOf),
-                        sentAt: null,
-                        receivedAt: Date.parse(item.receivedAt),
-                        bodyText: mail.text || null,
-                        size: item.size ?? stored.size,
-                        ...(Object.keys(header).length ? { headers: header } : {}),
-                        outgoing: mail.from === null ? null : mail.from.email === key.account,
-                        mailboxes: stored.mailboxes.map(({ externalId, name, kind }) => ({ externalId, name, kind })),
-                        ...(stored.read === null ? {} : { read: stored.read }),
-                        ...(stored.flagged === null ? {} : { flagged: stored.flagged }),
-                        metadata: stored.metadata,
-                        mailRuleId: stored.mailRuleId,
-                        attachments: files,
-                    },
-                ],
-            });
-            emails++;
-        }
-    }
-    const indexed = await index?.(store, key);
-    return {
-        account: key.account,
-        since: since.toISOString(),
-        listed: items.length,
-        saved: 0,
-        alreadyStored: 0,
-        skipped: 0,
-        envelopes: 0,
-        deleted: 0,
-        complete: wanted.length <= max && unreadable.length === 0,
-        ...(indexed === undefined ? {} : { indexed }),
-        retried: { emails, extracted, stillFailed, missing: chosen.length - items.length },
-        ...(attachments.length ? { attachments } : {}),
-        ...(unreadable.length ? { unreadable } : {}),
-    };
-};

package/dist/mail/testing/contract.js

@@ -53,6 +53,28 @@ export const mailSourceContract = (make) => {
             });
             assert.deepEqual(listed.map(({ uid }) => uid), first === undefined ? [] : [first]);
         }),
+        test("lists quickly the same UIDs, flags and sizes, and gives from the bytes the ids `list` gives", async (source) => {
+            if (source.search === undefined || source.quick === undefined || source.identify === undefined)
+                return;
+            const searched = await source.search(SINCE, folder);
+            if (!searched.quick)
+                return;
+            const quick = await source.quick(SINCE, folder, { searched });
+            const full = await source.list(SINCE, folder);
+            assert.deepEqual(quick.map(({ uid }) => uid).sort(), full.map(({ uid }) => uid).sort());
+            const raws = await source.fetchRaw(quick);
+            for (const [index, item] of quick.entries()) {
+                const raw = raws[index];
+                assert.ok(raw instanceof Uint8Array, "bytes");
+                const same = full.find(({ uid }) => uid === item.uid);
+                assert.deepEqual(source.identify(raw, item), {
+                    messageId: same?.messageId,
+                    threadId: same?.threadId,
+                });
+                assert.equal(item.seen, same?.seen);
+                assert.equal(item.receivedAt, same?.receivedAt);
+            }
+        }),
         test("returns each message's bytes in order, and one that cannot be read as an Error, not a failed call", async (source) => {
             const listed = await source.list(SINCE, folder);
             const raws = await source.fetchRaw(listed);

package/dist/mail/testing/fake.d.ts

@@ -26,6 +26,8 @@ export interface FakeMail {
     flagged?: boolean;
     /** `trash` puts it in the trash folder; otherwise it is in All Mail, which INBOX also shows. */
     folder?: "trash";
+    /** Folders that do not hold it although its `folder` says they do, as a server's delete from one folder leaves it. */
+    absentFrom?: string[];
 }
 /** The mailbox a fake source reads live, so a test changes it between syncs; the switches make the source fail. */
 export interface FakeMailbox {
@@ -48,7 +50,12 @@ export interface FakeMailbox {
         search: number;
         list: number;
         fetchRaw: number;
+        quick?: number;
     };
+    /** Answers as a server with no cheap message id does: `search` says `quick`, and `quick` and `identify` work. */
+    quick?: boolean;
+    /** The UIDs `fetchRaw` was asked for, in order, so a test proves nothing is read twice. */
+    read?: string[];
     /** What the source tells a budgeted sync about its batch sizes. */
     listBatch?: number;
     fetchBatch?: number;

package/dist/mail/testing/fake.js

@@ -29,18 +29,21 @@ export const rawOf = (m) => {
 };
 const failure = (message) => new CliError("provider_error", message, { reason: "mail.source-failed" });
 const labelsOf = (m) => [...new Set([...(m.labels ?? []), ...(m.flagged ? ["Starred"] : [])])].sort();
-const listingOf = (m, headers, folder) => ({
+const quickOf = (m, folder) => ({
     uid: m.uid,
-    messageId: m.messageId,
-    threadId: m.threadId,
     receivedAt: new Date(m.receivedAt).toISOString(),
     size: rawOf(m).length,
     labels: labelsOf(m),
     seen: m.seen ?? false,
     flagged: m.flagged ?? false,
-    ...(headers ? { headers: headersOf(Buffer.from(rawOf(m)).toString().split("\r\n\r\n")[0] ?? "") } : {}),
     folder,
 });
+const listingOf = (m, headers, folder) => ({
+    ...quickOf(m, folder),
+    messageId: m.messageId,
+    threadId: m.threadId,
+    ...(headers ? { headers: headersOf(Buffer.from(rawOf(m)).toString().split("\r\n\r\n")[0] ?? "") } : {}),
+});
 /**
  * An in-memory mail source with Gmail's shape: ids and threads that outlive a folder, labels and flags, UIDVALIDITY,
  * a trash a move reaches, and All Mail under the account's language. The sync tests and the adapter contract run on it.
@@ -51,7 +54,7 @@ export const fakeMailSource = (mailbox) => {
     const inFolder = (folder) => {
         if (![all(), trash(), "INBOX"].includes(folder))
             throw failure(`Gmail has no such mailbox: ${folder}`);
-        return mailbox.messages.filter((m) => (folder === trash() ? m.folder === "trash" : m.folder !== "trash"));
+        return mailbox.messages.filter((m) => (folder === trash() ? m.folder === "trash" : m.folder !== "trash") && !m.absentFrom?.includes(folder));
     };
     const search = async (since, folder) => {
         if (mailbox.searchFails)
@@ -59,6 +62,7 @@ export const fakeMailSource = (mailbox) => {
         const from = Date.UTC(since.getUTCFullYear(), since.getUTCMonth(), since.getUTCDate());
         return {
             validity: String(mailbox.validity ?? 12),
+            ...(mailbox.quick ? { quick: true } : {}),
             uids: inFolder(folder)
                 .filter((m) => Date.parse(m.receivedAt) >= from)
                 .map((m) => m.uid),
@@ -73,7 +77,7 @@ export const fakeMailSource = (mailbox) => {
     };
     const count = (kind) => {
         if (mailbox.calls)
-            mailbox.calls[kind] += 1;
+            mailbox.calls[kind] = (mailbox.calls[kind] ?? 0) + 1;
     };
     return {
         ...(mailbox.listBatch === undefined ? {} : { listBatch: mailbox.listBatch }),
@@ -94,6 +98,24 @@ export const fakeMailSource = (mailbox) => {
                 .filter((m) => wanted.has(m.uid) && !failsNow(m.uid))
                 .map((m) => listingOf(m, options?.headers ?? false, folder));
         },
+        ...(mailbox.quick
+            ? {
+                quick: async (_since, folder, options) => {
+                    count("quick");
+                    if (mailbox.searchFails)
+                        throw failure(mailbox.searchFails);
+                    const wanted = new Set(options.searched.uids);
+                    return inFolder(folder)
+                        .filter((m) => wanted.has(m.uid) && !failsNow(m.uid))
+                        .map((m) => quickOf(m, folder));
+                },
+                identify: (raw) => {
+                    const id = /^Message-ID:\s*<(.+?)@example\.test>/im.exec(Buffer.from(raw).toString())?.[1];
+                    const m = mailbox.messages.find((each) => each.messageId === id);
+                    return { messageId: m.messageId, threadId: m.threadId };
+                },
+            }
+            : {}),
         find: async (messageIds, folder, options) => {
             for (const id of messageIds)
                 mailbox.asked?.push(`msgid:${id}`);
@@ -131,6 +153,7 @@ export const fakeMailSource = (mailbox) => {
         },
         fetchRaw: async (items) => {
             count("fetchRaw");
+            mailbox.read?.push(...items.map(({ uid }) => uid));
             return items.map((item) => {
                 const m = mailbox.messages.find((each) => each.uid === item.uid);
                 if (m === undefined || mailbox.unreadable?.includes(item.uid) || failsNow(item.uid))

package/dist/store/index.d.ts

@@ -29,7 +29,7 @@ export type { BotUpdate, BotUpdateStore } from "./sqlite/bot-updates.js";
 export type { CatalogActor, CatalogEntry, CatalogInput, CatalogQuery, CatalogStore } from "./sqlite/catalog.js";
 export { CATALOG_QUERY } from "./sqlite/catalog.js";
 export type { ChatMetadata } from "./sqlite/chats.js";
-export type { Email, EmailAddress, EmailAttachment, EmailFacts, EmailFilter, EmailInput, EmailInThread, EmailRecipient, EmailRef, EmailThread, EmailVectorHit, Mailbox, MailboxInput, MailFilter, MailStore, SenderScore, ThreadDetails, ThreadSave, } from "./sqlite/emails.js";
+export type { Email, EmailAddress, EmailAttachment, EmailFacts, EmailFilter, EmailInput, EmailInThread, EmailLocationInput, EmailRecipient, EmailRef, EmailThread, EmailVectorHit, FolderEmail, Mailbox, MailboxInput, MailFilter, MailStore, SenderScore, ThreadDetails, ThreadSave, } from "./sqlite/emails.js";
 export { EVENT_FIELDS, type EventChunkToEmbed, type EventHit, type EventQuery, type EventSearch, type NearestEvent, } from "./sqlite/event-search.js";
 export type { Fact, FactInput, FactsStore } from "./sqlite/facts.js";
 export type { FileListQuery, FilesStore, StoredFile } from "./sqlite/files.js";

package/dist/store/sqlite/accounts.js

@@ -92,6 +92,7 @@ export const purgeAccount = ({ orm, database }, accountKey) => {
     orm.delete(messages).where(eq(messages.accountId, accountKey)).run();
     orm.delete(chats).where(eq(chats.accountId, accountKey)).run();
     orm.delete(syncCursors).where(eq(syncCursors.accountId, accountKey)).run();
+    orm.run(sql `DELETE FROM email_locations WHERE mailbox_id IN (SELECT id FROM mailboxes WHERE account_id = ${accountKey})`);
     // Deleting an email clears its recipients, mailboxes, words, attachments and tags (trigger email_bd).
     orm.run(sql `DELETE FROM emails WHERE account_id = ${accountKey}`);
     orm.run(sql `DELETE FROM taggings WHERE taggable_type = 'email_thread' AND taggable_id IN (SELECT id FROM email_threads WHERE account_id = ${accountKey})`);

package/dist/store/sqlite/email-locations.d.ts

@@ -0,0 +1,24 @@
+import type { CacheDatabase } from "../driver.js";
+/** Where an IMAP server holds an email: `folder` is the mailbox's external id. */
+export interface EmailLocationInput {
+    folder: string;
+    uidvalidity: string;
+    uid: number;
+}
+/** One email of a folder as a sync sees it there. */
+export interface FolderEmail {
+    uid: number;
+    externalId: string;
+    receivedAt: number | null;
+    read: boolean | null;
+    flagged: boolean | null;
+    /** Its locations in every folder, and the folders it is a member of: an email is gone only when all are lost. */
+    locations: number;
+    folders: number;
+}
+export declare const locate: (database: CacheDatabase, accountId: number, emailId: number, where: EmailLocationInput) => {
+    changes: number;
+};
+export declare const folderEmails: (database: CacheDatabase, accountId: number, folder: string, uidvalidity: string) => FolderEmail[];
+export declare const setLocations: (database: CacheDatabase, accountId: number, folder: string, uidvalidity: string, found: ReadonlyMap<string, number[]>) => number;
+export declare const forgetLocations: (database: CacheDatabase, accountId: number, folder: string, uidvalidity: string, uids: number[]) => void;

package/dist/store/sqlite/email-locations.js

@@ -0,0 +1,56 @@
+export const locate = (database, accountId, emailId, where) => database
+    .prepare(`INSERT OR REPLACE INTO email_locations (mailbox_id, uidvalidity, uid, email_id)
+         SELECT id, ?, ?, ? FROM mailboxes WHERE account_id = ? AND external_id = ?`)
+    .run(where.uidvalidity, where.uid, emailId, accountId, where.folder);
+export const folderEmails = (database, accountId, folder, uidvalidity) => database
+    .prepare(`SELECT l.uid, e.external_id, coalesce(e.received_at, e.sent_at) AS at, e.read, e.flagged,
+         (SELECT count(*) FROM email_locations x WHERE x.email_id = e.id) AS locations,
+         (SELECT count(*) FROM email_mailboxes m JOIN mailboxes b ON b.id = m.mailbox_id
+           WHERE m.email_id = e.id AND b.kind IS NULL) AS folders
+       FROM email_locations l JOIN emails e ON e.id = l.email_id
+       WHERE l.mailbox_id = (SELECT id FROM mailboxes WHERE account_id = ? AND external_id = ?)
+         AND l.uidvalidity = ? AND e.deleted_at IS NULL
+       ORDER BY l.uid`)
+    .all(accountId, folder, uidvalidity)
+    .map((row) => ({
+    uid: Number(row.uid),
+    externalId: String(row.external_id),
+    receivedAt: row.at === null ? null : Number(row.at),
+    read: row.read === null ? null : Boolean(row.read),
+    flagged: row.flagged === null ? null : Boolean(row.flagged),
+    locations: Number(row.locations),
+    folders: Number(row.folders),
+}));
+export const setLocations = (database, accountId, folder, uidvalidity, found) => {
+    database
+        .prepare(`DELETE FROM email_locations WHERE uidvalidity <> ?
+         AND mailbox_id = (SELECT id FROM mailboxes WHERE account_id = ? AND external_id = ?)`)
+        .run(uidvalidity, accountId, folder);
+    const lookup = database.prepare("SELECT id FROM emails WHERE account_id = ? AND external_id = ?");
+    let count = 0;
+    for (const [externalId, uids] of found) {
+        const row = lookup.get(accountId, externalId);
+        if (!row)
+            continue;
+        for (const uid of uids)
+            locate(database, accountId, Number(row.id), { folder, uidvalidity, uid });
+        count++;
+    }
+    return count;
+};
+export const forgetLocations = (database, accountId, folder, uidvalidity, uids) => {
+    const box = database
+        .prepare("SELECT id FROM mailboxes WHERE account_id = ? AND external_id = ?")
+        .get(accountId, folder);
+    if (!box)
+        return;
+    const emails = database
+        .prepare(`DELETE FROM email_locations WHERE mailbox_id = ? AND uidvalidity = ?
+         AND uid IN (SELECT value FROM json_each(?)) RETURNING email_id`)
+        .all(Number(box.id), uidvalidity, JSON.stringify(uids))
+        .map((row) => Number(row.email_id));
+    const leave = database.prepare(`DELETE FROM email_mailboxes WHERE email_id = ? AND mailbox_id = ?
+       AND NOT EXISTS (SELECT 1 FROM email_locations WHERE email_id = ? AND mailbox_id = ?)`);
+    for (const emailId of new Set(emails))
+        leave.run(emailId, Number(box.id), emailId, Number(box.id));
+};