@wirecat/cli-core 0.546.0 → 0.547.0

package/dist/mail/partial.js

@@ -0,0 +1,94 @@
+import { FETCH_CHUNK, readAttachments } from "./attachments.js";
+import { parsedEmail } from "./email-input.js";
+import { mailHeadersOf } from "./headers.js";
+import { parseMail } from "./parse.js";
+/** The `emails.metadata` key of an email stored from its first bytes: `{ bytes }` read of its `size`. */
+export const PARTIAL = "partial";
+/** How many partial emails one sync reads whole at most; the rest wait for the next run. */
+export const PARTIAL_PER_RUN = 100;
+/**
+ * Reads whole the emails a server search stored from their first bytes, so the archive holds their attachments too.
+ * Sync never reads stored mail again, so without this step a partial email would stay partial. Gmail's are found by
+ * their id; elsewhere at a stored location, trusted only when the bytes read there carry the stored id.
+ */
+export const completePartial = async ({ store, source, accountId, address, folders, maxBodyChars, maxAttachmentChars, loadEngine, now, take, signal, }) => {
+    const { emails, total } = await store.mail.partialEmails(accountId, PARTIAL_PER_RUN);
+    if (emails.length === 0)
+        return { completed: 0, left: 0 };
+    const byId = new Map(emails.map((email) => [email.externalId, email]));
+    const items = new Map();
+    if (source.find) {
+        for (const folder of await folders()) {
+            const left = emails.filter((email) => !items.has(email)).map(({ externalId }) => externalId);
+            if (left.length === 0)
+                break;
+            if (take?.())
+                return { completed: 0, left: total };
+            for (const found of await source.find(left, folder)) {
+                const email = byId.get(found.messageId);
+                if (email && !items.has(email))
+                    items.set(email, found);
+            }
+        }
+    }
+    else if (source.identify)
+        for (const email of emails) {
+            const place = email.locations[0];
+            if (place)
+                items.set(email, {
+                    uid: String(place.uid),
+                    folder: place.folder,
+                    receivedAt: new Date(email.receivedAt ?? 0).toISOString(),
+                });
+        }
+    let completed = 0;
+    const wanted = [...items];
+    for (let start = 0; start < wanted.length; start += FETCH_CHUNK) {
+        if (signal?.aborted || take?.())
+            break;
+        const chunk = wanted.slice(start, start + FETCH_CHUNK);
+        let raws;
+        try {
+            raws = await source.fetchRaw(chunk.map(([, item]) => item));
+        }
+        catch {
+            // A partial email stays readable as it is; the next run tries again.
+            break;
+        }
+        const attachments = [];
+        for (const [index, [email, item]] of chunk.entries()) {
+            const raw = raws[index];
+            if (!(raw instanceof Uint8Array))
+                continue;
+            // A UID means another message after a UIDVALIDITY change; only the stored id proves it is this one.
+            if (!source.find && source.identify?.(raw, item).messageId !== email.externalId)
+                continue;
+            try {
+                const mail = await parseMail(raw, maxBodyChars);
+                const files = await readAttachments(mail, email.externalId, { loadEngine, maxAttachmentChars, at: now, ...(signal ? { signal } : {}) }, attachments);
+                const { [PARTIAL]: _partial, ...metadata } = email.metadata ?? {};
+                const whole = { ...item, messageId: email.externalId, threadId: email.threadExternalId };
+                await store.mail.saveThread({
+                    accountId,
+                    externalId: email.threadExternalId,
+                    now,
+                    emails: [
+                        {
+                            ...parsedEmail(mail, whole, mailHeadersOf(raw), address),
+                            receivedAt: email.receivedAt,
+                            size: raw.byteLength,
+                            metadata,
+                            attachments: files,
+                        },
+                    ],
+                });
+                completed++;
+            }
+            catch {
+                if (signal?.aborted)
+                    break;
+            }
+        }
+    }
+    return { completed, left: total - completed };
+};

package/dist/mail/server-search.js

@@ -9,6 +9,7 @@ import { parsedEmail } from "./email-input.js";
 import { factsOf } from "./facts.js";
 import { mailHeadersOf } from "./headers.js";
 import { parseMail } from "./parse.js";
+import { PARTIAL } from "./partial.js";
 import { storedMailRules } from "./rule-stored.js";
 import { ruleTagger, settleRuleTags } from "./rule-tags.js";
 import { decide, tagsOf } from "./rules.js";
@@ -84,6 +85,7 @@ export const searchMailServer = async (store, mailbox, access, request) => {
     const ruled = new Map();
     const inputs = [];
     const saved = [];
+    let cutShort = 0;
     let opening;
     let openFailure;
     let unsupported = false;
@@ -146,17 +148,12 @@ export const searchMailServer = async (store, mailbox, access, request) => {
             sources.set(sourceKey(item.threadId, item.messageId), held ? "both" : "server");
             if (held)
                 continue;
-            // A big message would hold the connection and the parser past the bound; sync reads it whole later.
-            if ((item.size ?? 0) > SERVER_BOUNDS.maxMessageBytes) {
-                report.tooLarge = (report.tooLarge ?? 0) + 1;
-                report.complete = false;
-                sources.delete(sourceKey(item.threadId, item.messageId));
-                continue;
-            }
             fresh.push(item);
             ruled.set(item, { ruleId: rule?.id ?? null, tags: tagsOf(rules, facts(item)) });
         }
-        const raws = fresh.length && open ? await source.fetchRaw(fresh) : [];
+        // A big message would hold the connection and the parser past the bound: its first bytes carry the text.
+        const bytes = SERVER_BOUNDS.textBytes;
+        const raws = fresh.length && open ? await source.fetchRaw(fresh, { bytes }) : [];
         // Parsing and reading attachments stay inside the bound: after it, nothing more is read or kept.
         const now = Date.now();
         for (const [index, item] of fresh.entries()) {
@@ -166,8 +163,12 @@ export const searchMailServer = async (store, mailbox, access, request) => {
             try {
                 if (raw === undefined || raw instanceof Error)
                     throw raw ?? new Error("not read");
+                const cut = raw.byteLength >= bytes && (item.size === undefined || item.size > raw.byteLength);
                 const mail = await parseMail(raw);
-                const files = await readAttachments(mail, item.messageId, { loadEngine: importEngine, maxAttachmentChars: MAX_ATTACHMENT_CHARS, at: now, signal: stop }, []);
+                // An attachment cut short is not the file: none is kept until the sync reads the whole message.
+                const files = cut
+                    ? []
+                    : await readAttachments(mail, item.messageId, { loadEngine: importEngine, maxAttachmentChars: MAX_ATTACHMENT_CHARS, at: now, signal: stop }, []);
                 if (!open)
                     return;
                 const folder = item.folder;
@@ -189,12 +190,15 @@ export const searchMailServer = async (store, mailbox, access, request) => {
                                 : { locations: [{ folder, uidvalidity: validity, uid: Number(item.uid) }] }),
                             ...(item.seen === undefined ? {} : { read: item.seen }),
                             ...(item.flagged === undefined ? {} : { flagged: item.flagged }),
+                            ...(cut ? { metadata: { [PARTIAL]: { bytes: raw.byteLength } } } : {}),
                             mailRuleId: ruled.get(item)?.ruleId ?? null,
                             attachments: files,
                         },
                     ],
                 });
                 saved.push(item);
+                if (cut)
+                    cutShort++;
             }
             catch {
                 sources.delete(sourceKey(item.threadId, item.messageId));
@@ -248,6 +252,10 @@ export const searchMailServer = async (store, mailbox, access, request) => {
     }
     await settleRuleTags(store, accountId, rules, tagger.outcome());
     report.new = inputs.length;
+    if (cutShort) {
+        report.partial = cutShort;
+        report.complete = false;
+    }
     report.returned = sources.size;
     return { report, sources };
 };

package/dist/mail/source.d.ts

@@ -1,6 +1,12 @@
 import type { MailHeaders } from "./headers.js";
 import type { FileStamp } from "./resume.js";
 import type { MailCriteria } from "./server-query.js";
+/**
+ * The reason of the CliError `search`, `query` and `present` throw when the server answers SEARCH with NO or BAD:
+ * read as "nothing found", it would look like an empty folder and prove its stored mail deleted.
+ */
+export declare const SEARCH_REFUSED = "mail.search-refused";
+export declare const searchRefused: (error: unknown) => boolean;
 export interface Listed {
     uid: string;
     /** Gmail's own ids: the same in every folder, and they survive a UID reset. */
@@ -94,6 +100,11 @@ export interface MailSource {
     present?(uids: readonly string[], folder: string): Promise<string[]>;
     /** Puts labels back on messages in `folder`, by Gmail id. */
     label?(messageIds: readonly string[], folder: string, labels: readonly string[]): Promise<void>;
-    /** Each item's raw RFC 5322 bytes, in order; one that could not be read is an Error, not a failed call. */
-    fetchRaw(items: Quick[]): Promise<(Uint8Array | Error)[]>;
+    /**
+     * Each item's raw RFC 5322 bytes, in order; one that could not be read is an Error, not a failed call. With
+     * `bytes`, at most the first `bytes` of each: a message cut short comes back shorter than its `size`.
+     */
+    fetchRaw(items: Quick[], options?: {
+        bytes?: number;
+    }): Promise<(Uint8Array | Error)[]>;
 }

package/dist/mail/source.js

@@ -1 +1,7 @@
-export {};
+import { CliError } from "@wirecat/cli-base";
+/**
+ * The reason of the CliError `search`, `query` and `present` throw when the server answers SEARCH with NO or BAD:
+ * read as "nothing found", it would look like an empty folder and prove its stored mail deleted.
+ */
+export const SEARCH_REFUSED = "mail.search-refused";
+export const searchRefused = (error) => error instanceof CliError && error.details.reason === SEARCH_REFUSED;

package/dist/mail/sync-types.d.ts

@@ -0,0 +1,112 @@
+import type { BudgetStop, RunBudget } from "../background/budget.js";
+import type { LoadEngine } from "../readers/types.js";
+import type { AccountKey, MessageStore } from "../store/index.js";
+import type { PartialOutcome } from "./partial.js";
+import type { TagOutcome } from "./rule-tags.js";
+import type { MailRule } from "./rules.js";
+import type { DomainAge } from "./score.js";
+export interface MailAccount {
+    /** The mailbox address: the store's account id. */
+    address: string;
+    mode?: "gmail" | "imap" | "mbox";
+    folders?: string[];
+    rules?: MailRule[];
+}
+/** What an index step reports, as memo's embedding does; the sync only passes it on. */
+export interface MailIndexed {
+    chats: number;
+    chunks: number;
+    left: boolean;
+    notEmbedded?: unknown;
+}
+export interface MailSyncResult {
+    account: string;
+    since: string;
+    listed: number;
+    saved: number;
+    alreadyStored: number;
+    /** New emails a skip rule kept out: never read, never stored. */
+    skipped: number;
+    /** New emails an envelope rule recorded: sender, date, size and labels, never their text. */
+    envelopes: number;
+    deleted: number;
+    /** False when `max` stopped the run before every new message was read; the next run continues. */
+    complete: boolean;
+    /** `partial`: a budgeted run stopped before the whole window was read; the next run goes on. */
+    state?: "partial" | "complete";
+    /** Why a budgeted run stopped early: its request budget, its deadline, or a cancel. */
+    stopped?: BudgetStop;
+    /** Messages of the window this run did not get to read: left by a budget, or by `max`. */
+    remaining?: number;
+    /** Requests a budgeted run spent. */
+    requests?: number;
+    /** Emails that could not be read or parsed; they are tried again next run. */
+    unreadable?: {
+        message: string;
+        reason: string;
+    }[];
+    /** Why stored messages missing at the source were left alone, when they were. */
+    deletionsSkipped?: string;
+    /** Folders whose server refused SEARCH: left unread, so the run is partial and proves no deletion. */
+    refusedFolders?: string[];
+    /** "resumed": only mail not done before was listed; "full": every message in the window was. */
+    pass?: "full" | "resumed";
+    indexed?: MailIndexed;
+    coverage?: {
+        folders: string[];
+        since: string;
+        until: string;
+        state: "window" | "partial";
+        identity: string;
+    };
+    attachments?: {
+        message: string;
+        part: number;
+        status: string;
+    }[];
+    /** Rule tags put on mail as an existing label the store pointed to. */
+    tagsReused?: TagOutcome["reused"];
+    /** Rule tags the store refused with nothing to reuse; that mail is stored untagged. */
+    tagsSkipped?: TagOutcome["skipped"];
+    /** Emails a server search stored from their first bytes, read whole this run, and those still partial. */
+    partial?: PartialOutcome;
+    /** `--retry-attachments`: emails read again for attachments a missing reader left unread. */
+    retried?: {
+        emails: number;
+        extracted: number;
+        stillFailed: number;
+        missing: number;
+    };
+}
+export interface MailSyncOptions {
+    account: MailAccount;
+    since: Date;
+    max?: number;
+    maxBodyChars?: number;
+    maxAttachmentChars?: number;
+    now?: () => number;
+    /** Domain registration dates already known, for the sender score a rule asks for. */
+    ages?: ReadonlyMap<string, DomainAge>;
+    loadEngine?: LoadEngine;
+    retryAttachments?: boolean;
+    /** List every message in the window, not only those an earlier run left: proves deletions, refreshes labels. */
+    rescan?: boolean;
+    /** `false` for a spot read, like the copy of a sent message: the run leaves the saved coverage as the last full run set it. */
+    saveCoverage?: boolean;
+    /** A window smaller than this is listed whole every run: it costs seconds and keeps flags and deletions current. */
+    resumeFrom?: number;
+    /** Runs after the mail is saved, so chats and mail still waiting for vectors are indexed; its answer is `indexed`. */
+    index?: (store: MessageStore, key: AccountKey) => Promise<MailIndexed>;
+    /** Stops between batches of emails, after saving the resume point. */
+    signal?: AbortSignal;
+    /**
+     * Makes the run budgeted: it reads the window in rounds, one request of the budget per call to the source, saves
+     * after each, stops when the budget says so and answers `state: "partial"`. A message the source could not give
+     * is tried once more at the end of the run and, if it still fails, left for the next one. `max` defaults to no limit.
+     */
+    budget?: RunBudget;
+    /** Pauses before a round is read again after the store was locked; tests pass one that does not wait. */
+    wait?: (ms: number) => Promise<void>;
+    /** `false` for a spot read: emails a server search stored from their first bytes are not read whole this run. */
+    completePartial?: boolean;
+}

package/dist/mail/sync-types.js

@@ -0,0 +1 @@
+export {};

package/dist/mail/sync.d.ts

@@ -1,109 +1,7 @@
-import type { BudgetStop, RunBudget } from "../background/budget.js";
-import type { LoadEngine } from "../readers/types.js";
-import type { AccountKey, MessageStore } from "../store/index.js";
-import { type TagOutcome } from "./rule-tags.js";
-import { type MailRule } from "./rules.js";
-import type { DomainAge } from "./score.js";
-import type { MailSource } from "./source.js";
-export interface MailAccount {
-    /** The mailbox address: the store's account id. */
-    address: string;
-    mode?: "gmail" | "imap" | "mbox";
-    folders?: string[];
-    rules?: MailRule[];
-}
-/** What an index step reports, as memo's embedding does; the sync only passes it on. */
-export interface MailIndexed {
-    chats: number;
-    chunks: number;
-    left: boolean;
-    notEmbedded?: unknown;
-}
-export interface MailSyncResult {
-    account: string;
-    since: string;
-    listed: number;
-    saved: number;
-    alreadyStored: number;
-    /** New emails a skip rule kept out: never read, never stored. */
-    skipped: number;
-    /** New emails an envelope rule recorded: sender, date, size and labels, never their text. */
-    envelopes: number;
-    deleted: number;
-    /** False when `max` stopped the run before every new message was read; the next run continues. */
-    complete: boolean;
-    /** `partial`: a budgeted run stopped before the whole window was read; the next run goes on. */
-    state?: "partial" | "complete";
-    /** Why a budgeted run stopped early: its request budget, its deadline, or a cancel. */
-    stopped?: BudgetStop;
-    /** Messages of the window this run did not get to read: left by a budget, or by `max`. */
-    remaining?: number;
-    /** Requests a budgeted run spent. */
-    requests?: number;
-    /** Emails that could not be read or parsed; they are tried again next run. */
-    unreadable?: {
-        message: string;
-        reason: string;
-    }[];
-    /** Why stored messages missing at the source were left alone, when they were. */
-    deletionsSkipped?: string;
-    /** "resumed": only mail not done before was listed; "full": every message in the window was. */
-    pass?: "full" | "resumed";
-    indexed?: MailIndexed;
-    coverage?: {
-        folders: string[];
-        since: string;
-        until: string;
-        state: "window" | "partial";
-        identity: string;
-    };
-    attachments?: {
-        message: string;
-        part: number;
-        status: string;
-    }[];
-    /** Rule tags put on mail as an existing label the store pointed to. */
-    tagsReused?: TagOutcome["reused"];
-    /** Rule tags the store refused with nothing to reuse; that mail is stored untagged. */
-    tagsSkipped?: TagOutcome["skipped"];
-    /** `--retry-attachments`: emails read again for attachments a missing reader left unread. */
-    retried?: {
-        emails: number;
-        extracted: number;
-        stillFailed: number;
-        missing: number;
-    };
-}
+import type { MessageStore } from "../store/index.js";
+import { type MailSource } from "./source.js";
+import type { MailSyncOptions, MailSyncResult } from "./sync-types.js";
+export type { MailAccount, MailIndexed, MailSyncOptions, MailSyncResult } from "./sync-types.js";
 export { MAX_ATTACHMENT_CHARS } from "./attachments.js";
 export { RETRYABLE_ATTACHMENTS } from "./retry.js";
-export interface MailSyncOptions {
-    account: MailAccount;
-    since: Date;
-    max?: number;
-    maxBodyChars?: number;
-    maxAttachmentChars?: number;
-    now?: () => number;
-    /** Domain registration dates already known, for the sender score a rule asks for. */
-    ages?: ReadonlyMap<string, DomainAge>;
-    loadEngine?: LoadEngine;
-    retryAttachments?: boolean;
-    /** List every message in the window, not only those an earlier run left: proves deletions, refreshes labels. */
-    rescan?: boolean;
-    /** `false` for a spot read, like the copy of a sent message: the run leaves the saved coverage as the last full run set it. */
-    saveCoverage?: boolean;
-    /** A window smaller than this is listed whole every run: it costs seconds and keeps flags and deletions current. */
-    resumeFrom?: number;
-    /** Runs after the mail is saved, so chats and mail still waiting for vectors are indexed; its answer is `indexed`. */
-    index?: (store: MessageStore, key: AccountKey) => Promise<MailIndexed>;
-    /** Stops between batches of emails, after saving the resume point. */
-    signal?: AbortSignal;
-    /**
-     * Makes the run budgeted: it reads the window in rounds, one request of the budget per call to the source, saves
-     * after each, stops when the budget says so and answers `state: "partial"`. A message the source could not give
-     * is tried once more at the end of the run and, if it still fails, left for the next one. `max` defaults to no limit.
-     */
-    budget?: RunBudget;
-    /** Pauses before a round is read again after the store was locked; tests pass one that does not wait. */
-    wait?: (ms: number) => Promise<void>;
-}
 export declare const syncMail: (source: MailSource, store: MessageStore, options: MailSyncOptions) => Promise<MailSyncResult>;

package/dist/mail/sync.js

@@ -12,11 +12,13 @@ import { dropMissing, tooManyGone } from "./gone.js";
 import { mailHeadersOf } from "./headers.js";
 import { flagsToRefresh, proveByLocation } from "./locations.js";
 import { MAX_BODY_CHARS, parseMail } from "./parse.js";
+import { completePartial } from "./partial.js";
 import { FETCH_SLICE, keyOf, readQuick } from "./quick.js";
 import { FULL_PASS_MS, MBOX_RESUME, mergeRanges, parsePoint, rangesOf, resumeKey, rulesKey, within, } from "./resume.js";
 import { retryUnread } from "./retry.js";
 import { ruleTagger, settleRuleTags } from "./rule-tags.js";
 import { decide, ENVELOPE, envelopeBox, envelopeBoxOf, tagsOf } from "./rules.js";
+import { searchRefused } from "./source.js";
 const DAY = 86_400_000;
 /** IMAP's SINCE counts whole days in the server's zone, so the window's first days are not trusted. */
 const EDGE = 2 * DAY;
@@ -32,6 +34,29 @@ const LIST_SLICE = 200;
 const STORE_BUSY_TRIES = 5;
 const STORE_BUSY_WAIT_MS = 2_000;
 export const syncMail = async (source, store, options) => {
+    const result = await syncRounds(source, store, options);
+    if (options.completePartial === false || options.retryAttachments || options.account.mode === "mbox")
+        return result;
+    if (result.stopped !== undefined || options.signal?.aborted)
+        return result;
+    const partial = await completePartial({
+        store,
+        source,
+        accountId: await store.accounts.saveAccount(mailAccountKey(options.account.address), {
+            name: options.account.address,
+        }),
+        address: options.account.address,
+        folders: () => mailFolders(options.account, source),
+        maxBodyChars: options.maxBodyChars ?? MAX_BODY_CHARS,
+        maxAttachmentChars: options.maxAttachmentChars ?? MAX_ATTACHMENT_CHARS,
+        loadEngine: options.loadEngine ?? importEngine,
+        now: (options.now ?? Date.now)(),
+        ...(options.budget ? { take: () => options.budget?.take() ?? null } : {}),
+        ...(options.signal ? { signal: options.signal } : {}),
+    });
+    return partial.completed || partial.left ? { ...result, partial } : result;
+};
+const syncRounds = async (source, store, options) => {
     const { budget, index } = options;
     if (budget === undefined || options.retryAttachments || options.account.mode === "mbox")
         return syncRound(source, store, options);
@@ -45,6 +70,7 @@ export const syncMail = async (source, store, options) => {
         unlisted: new Set(),
         quick: new Map(),
         quickFolders: new Set(),
+        refused: new Set(),
         stopped: null,
         remaining: 0,
         asked: 0,
@@ -195,10 +221,16 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
     /** A quick folder's stored mail as it was before this run changed anything, with the UID of each. */
     const folderMail = new Map();
     const refreshed = new Map();
+    /** Folders whose server refused SEARCH: nothing of them is known, so nothing of them is proved or moved. */
+    const refused = new Set();
     const holdsMail = async (folder) => (await store.mail.emails({ accountId, mailbox: folder, limit: 1 })).length > 0;
     for (const folder of folders) {
         if (mboxPoint !== undefined)
             break;
+        if (plan?.refused.has(folder)) {
+            refused.add(folder);
+            continue;
+        }
         let searched;
         if (source.search !== undefined && !snapshot) {
             searched = plan?.searches.get(folder);
@@ -209,7 +241,16 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
                     capped = true;
                     continue;
                 }
-                searched = await source.search(since, folder);
+                try {
+                    searched = await source.search(since, folder);
+                }
+                catch (error) {
+                    if (!searchRefused(error))
+                        throw error;
+                    refused.add(folder);
+                    plan?.refused.add(folder);
+                    continue;
+                }
                 plan?.searches.set(folder, searched);
             }
         }
@@ -380,6 +421,8 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
     let skipped;
     if (snapshot)
         skipped = "an mbox file is a snapshot; mail missing from it is not proven deleted";
+    else if (refused.size > 0)
+        skipped = `the server refused to search ${[...refused].join(", ")}; no deletion is checked until it answers`;
     else if (plan !== undefined && (capped || plan.tried.size > plan.asked || plan.only !== undefined))
         skipped = "this run read the window in parts; a run that lists all of it at once checks deletions";
     else if (quickRan.size > 0) {
@@ -427,7 +470,7 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
         skipped ??= older.skipped;
     }
     if (!snapshot)
-        await store.mail.setMailboxes(accountId, folders.map(mailbox), new Map([...foundFolders].map(([id, found]) => [id, [...found]])), startedAt);
+        await store.mail.setMailboxes(accountId, folders.filter((folder) => !refused.has(folder)).map(mailbox), new Map([...foundFolders].map(([id, found]) => [id, [...found]])), startedAt);
     // A message's label list is whole, so every label the account has is replaced for each message read.
     if (labelsOf.size && !snapshot) {
         const known = (await store.mail.mailboxes(accountId)).filter(({ kind }) => kind === "label");
@@ -766,7 +809,7 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
                     ],
                 ]),
         ]), startedAt);
-    const incomplete = capped || (plan !== undefined && (plan.stopped !== null || plan.failed.size > 0));
+    const incomplete = capped || refused.size > 0 || (plan !== undefined && (plan.stopped !== null || plan.failed.size > 0));
     const complete = !full && !incomplete && unreadable.length === 0 && (plan?.unreadable.size ?? 0) === 0;
     const bulk = new Map(listed.filter((item) => item.headers?.bulk).map((item) => [item.messageId, [bulkBox.externalId]]));
     if (bulk.size)
@@ -782,7 +825,8 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
         state: complete ? "window" : "partial",
         identity: account.mode === "imap" ? "message-id-or-uidvalidity-folder-uid" : "gmail-message-id",
     };
-    if (!snapshot && saveCoverage)
+    // A refused folder says nothing of its mail, so the coverage an earlier run proved stays as it was.
+    if (!snapshot && saveCoverage && refused.size === 0)
         await store.sync.setSyncState(key, "mail_coverage", JSON.stringify(coverage));
     const tagging = tagger.outcome();
     await settleRuleTags(store, accountId, rules, tagging);
@@ -802,6 +846,7 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
         ...(attachments.length ? { attachments } : {}),
         ...(unreadable.length ? { unreadable } : {}),
         ...(skipped === undefined ? {} : { deletionsSkipped: skipped }),
+        ...(refused.size ? { refusedFolders: [...refused] } : {}),
         ...(searches.size > 0 ? { pass: resumed.size > 0 ? "resumed" : "full" } : {}),
         ...(tagging.reused.length ? { tagsReused: tagging.reused } : {}),
         ...(tagging.skipped.length ? { tagsSkipped: tagging.skipped } : {}),

package/dist/mail/testing/fake.d.ts

@@ -1,6 +1,6 @@
 import type { FileStamp } from "../resume.js";
 import type { MailCriteria } from "../server-query.js";
-import type { MailSource } from "../source.js";
+import { type MailSource } from "../source.js";
 export type FakePerson = [name: string | null, address: string];
 /** One message of a fake Gmail-like mailbox: its ids, envelope and the parts `rawOf` writes as RFC 5322. */
 export interface FakeMail {
@@ -39,6 +39,8 @@ export interface FakeMailbox {
     validity?: number;
     /** Each answer of `search` and `list` ends in this failure. */
     searchFails?: string;
+    /** Folders whose SEARCH the server refuses: `search` and `query` throw `SEARCH_REFUSED`. */
+    refusesSearch?: string[];
     /** A refused header fetch fails the whole listing. */
     headersFail?: boolean;
     moveFails?: boolean;

package/dist/mail/testing/fake.js

@@ -1,6 +1,7 @@
 import { CliError } from "@wirecat/cli-base";
 import { GMAIL_ALL_MAIL } from "../folders.js";
 import { headersOf } from "../headers.js";
+import { SEARCH_REFUSED } from "../source.js";
 /** As IMAP SEARCH matches: a case-blind substring of the field, every part required. */
 const matches = (m, criteria) => {
     const has = (field, value) => field.toLowerCase().includes(value.toLowerCase());
@@ -73,9 +74,14 @@ export const fakeMailSource = (mailbox) => {
             throw failure(`Gmail has no such mailbox: ${folder}`);
         return mailbox.messages.filter((m) => (folder === trash() ? m.folder === "trash" : m.folder !== "trash") && !m.absentFrom?.includes(folder));
     };
+    const refuse = (folder) => {
+        if (mailbox.refusesSearch?.includes(folder))
+            throw new CliError("provider_error", `the mail server refused to search ${folder}`, { reason: SEARCH_REFUSED });
+    };
     const search = async (since, folder) => {
         if (mailbox.searchFails)
             throw failure(mailbox.searchFails);
+        refuse(folder);
         const from = Date.UTC(since.getUTCFullYear(), since.getUTCMonth(), since.getUTCDate());
         return {
             validity: String(mailbox.validity ?? 12),
@@ -137,6 +143,7 @@ export const fakeMailSource = (mailbox) => {
             count("search");
             if (mailbox.searchFails)
                 throw failure(mailbox.searchFails);
+            refuse(folder);
             mailbox.queried?.push({ criteria, folder });
             return {
                 validity: String(mailbox.validity ?? 12),
@@ -180,14 +187,15 @@ export const fakeMailSource = (mailbox) => {
                 if (messageIds.includes(m.messageId))
                     m.labels = [...new Set([...(m.labels ?? []), ...labels])];
         },
-        fetchRaw: async (items) => {
+        fetchRaw: async (items, options) => {
             count("fetchRaw");
             mailbox.read?.push(...items.map(({ uid }) => uid));
             return items.map((item) => {
                 const m = mailbox.messages.find((each) => each.uid === item.uid);
                 if (m === undefined || mailbox.unreadable?.includes(item.uid) || failsNow(item.uid))
                     return failure(`email ${item.uid} was not read`);
-                return rawOf(m);
+                const raw = rawOf(m);
+                return options?.bytes === undefined ? raw : raw.subarray(0, options.bytes);
             });
         },
     };

package/dist/services/deps.d.ts

@@ -70,8 +70,6 @@ export declare const storeIfOpen: (deps: ServiceDeps) => Promise<{
 export declare const fromStore: (deps: Pick<ServiceDeps, "offline" | "reads">) => boolean;
 /** What a store-mode read answers for a chat the store holds nothing of. */
 export declare const nothingStored: (messenger: MessengerTraits) => string;
-/** Where a store-mode messenger's history comes from, for what it cannot do. */
-export declare const PUSHED = "this messenger's history is read from the local store";
 export declare const OFFLINE = "--offline answers only from what is kept locally: `chats list|show`, `messages list|show|context|download`, `contacts list|show` and `review`";
 /** Over a connection that is already open — an MCP session holds one for minutes. */
 export declare const onlineDeps: (messenger: ServiceMessenger, adapter: MessengerAdapter, guard: SendGuard, where?: {

package/dist/services/deps.js

@@ -53,8 +53,6 @@ export const storeIfOpen = async (deps) => {
 export const fromStore = (deps) => deps.offline || deps.reads === "store";
 /** What a store-mode read answers for a chat the store holds nothing of. */
 export const nothingStored = (messenger) => `nothing stored for this chat yet — keep \`${typedCommand(messenger.app.command, "serve")}\` running`;
-/** Where a store-mode messenger's history comes from, for what it cannot do. */
-export const PUSHED = "this messenger's history is read from the local store";
 export const OFFLINE = "--offline answers only from what is kept locally: `chats list|show`, `messages list|show|context|download`, `contacts list|show` and `review`";
 /** Over a connection that is already open — an MCP session holds one for minutes. */
 export const onlineDeps = (messenger, adapter, guard, where = {}) => ({