@wirecat/cli-core 0.470.0 → 0.472.0

package/dist/mail/gone.js

@@ -0,0 +1,53 @@
+const PAGE = 500;
+const MAX_DELETED_SHARE = 0.2;
+const MIN_DELETIONS_CAPPED = 5;
+/** Why `gone` of `candidates` is too large a share to delete, or `undefined` when it may be deleted. */
+export const tooManyGone = (gone, candidates) => gone >= MIN_DELETIONS_CAPPED && gone > candidates * MAX_DELETED_SHARE
+    ? `${gone} of ${candidates} stored entries are missing at the source — too many to trust; nothing deleted`
+    : undefined;
+/**
+ * What the store holds of this account, from `since` on, and the source no longer lists, was deleted
+ * there: its text goes. Wrongly dropping text cannot be undone and a missed deletion is caught on the
+ * next run, so an unusually large share missing is refused rather than applied — an unmounted folder or
+ * a half-answered listing looks exactly like that.
+ */
+export const dropMissing = async (store, key, present, since, eligible) => {
+    const stored = [];
+    for (let offset = 0;; offset += PAGE) {
+        const chats = await store.chats.chats(key, { limit: PAGE, offset });
+        for (const chat of chats.items) {
+            if (since !== undefined && chat.lastMessageAt !== null && chat.lastMessageAt < since)
+                continue;
+            for (let before;;) {
+                const page = await store.messages.messages(key, chat.id, {
+                    limit: PAGE,
+                    ...(since === undefined ? {} : { since }),
+                    ...(before === undefined ? {} : { before }),
+                });
+                stored.push(...page.items.map(({ id }) => ({ chatId: chat.id, id })));
+                before = page.items[0]?.id;
+                if (!page.hasMore || before === undefined)
+                    break;
+            }
+        }
+        if (!chats.hasMore)
+            break;
+    }
+    const candidates = [];
+    for (const entry of stored)
+        if (eligible === undefined || (await eligible(entry)))
+            candidates.push(entry);
+    const gone = candidates.filter(({ id }) => !present.has(id));
+    if (gone.length === 0)
+        return { deleted: 0 };
+    const refused = tooManyGone(gone.length, candidates.length);
+    if (refused)
+        return { deleted: 0, skipped: refused };
+    const byChat = new Map();
+    for (const { chatId, id } of gone)
+        byChat.set(chatId, [...(byChat.get(chatId) ?? []), id]);
+    let deleted = 0;
+    for (const [chatId, ids] of byChat)
+        deleted += await store.messages.markDeleted(key, ids, { chatId });
+    return { deleted };
+};

package/dist/mail/headers.d.ts

@@ -0,0 +1,43 @@
+/**
+ * The header fields memo reads at list time: enough to group mail by sender, apply the owner's rules and keep
+ * an envelope of the mail a rule records without its text.
+ */
+export declare const HEADER_FIELDS = "FROM TO CC SUBJECT LIST-ID LIST-UNSUBSCRIBE PRECEDENCE REPLY-TO AUTHENTICATION-RESULTS CONTENT-TYPE";
+/** SPF, DKIM and DMARC as Gmail judged them when the email arrived; each absent when Gmail did not say. */
+export interface AuthVerdicts {
+    spf?: string;
+    dkim?: string;
+    dmarc?: string;
+}
+export interface MailHeaders {
+    /** The sender's address, lower case; absent when From holds none. */
+    from?: string;
+    /** The sender's display name, decoded. */
+    fromName?: string;
+    subject?: string;
+    /** To and Cc addresses, lower case. */
+    to: string[];
+    listId?: string;
+    /** List-Id, List-Unsubscribe or `Precedence: bulk|list|junk`: mail sent to many, not written to one. */
+    bulk: boolean;
+    /** The first Reply-To address, lower case. */
+    replyTo?: string;
+    auth?: AuthVerdicts;
+    /** `Content-Type: multipart/mixed`: how mail with attachments is sent. A guess, not a parse of the body. */
+    attachments?: boolean;
+}
+export declare const unfold: (text: string) => string;
+export declare const field: (unfolded: string, name: string) => string | undefined;
+/**
+ * The verdicts in Gmail's own Authentication-Results. Only the copy Gmail stamps as mx.google.com counts: a sender
+ * can add one under any other name, and Gmail removes forged copies under its own (RFC 8601 §5).
+ */
+export declare const authOf: (unfolded: string) => AuthVerdicts | undefined;
+export declare const addressesOf: (value: string | undefined) => string[];
+/** The header fields of one email, from its header text. */
+export declare const headersOf: (text: string) => MailHeaders;
+/**
+ * List-Id, the bulk-mail fields, Reply-To and Gmail's SPF, DKIM and DMARC verdicts of a whole email, kept on mail
+ * stored in full so a rule or a sender score can be worked out from the store later. Names are lower case.
+ */
+export declare const mailHeadersOf: (raw: Uint8Array) => Record<string, string | boolean>;

package/dist/mail/headers.js

@@ -0,0 +1,88 @@
+import { decodeWords } from "postal-mime";
+/**
+ * The header fields memo reads at list time: enough to group mail by sender, apply the owner's rules and keep
+ * an envelope of the mail a rule records without its text.
+ */
+export const HEADER_FIELDS = "FROM TO CC SUBJECT LIST-ID LIST-UNSUBSCRIBE PRECEDENCE REPLY-TO AUTHENTICATION-RESULTS CONTENT-TYPE";
+export const unfold = (text) => text.replace(/\r?\n[ \t]+/g, " ");
+export const field = (unfolded, name) => new RegExp(`^${name}:[ \\t]*(.*?)\\r?$`, "im").exec(unfolded)?.[1];
+const fieldsOf = (unfolded, name) => [...unfolded.matchAll(new RegExp(`^${name}:[ \\t]*(.*?)\\r?$`, "gim"))].map(([, value]) => value);
+/**
+ * The verdicts in Gmail's own Authentication-Results. Only the copy Gmail stamps as mx.google.com counts: a sender
+ * can add one under any other name, and Gmail removes forged copies under its own (RFC 8601 §5).
+ */
+export const authOf = (unfolded) => {
+    const stamped = fieldsOf(unfolded, "Authentication-Results").find((value) => /^mx\.google\.com\s*;/i.test(value));
+    if (stamped === undefined)
+        return undefined;
+    const results = [...stamped.matchAll(/\b(spf|dkim|dmarc)=([a-z]+)/gi)].map(([, method, result]) => [
+        method.toLowerCase(),
+        result.toLowerCase(),
+    ]);
+    const verdicts = {};
+    for (const [method, result] of results) {
+        const key = method;
+        // An email can carry several DKIM signatures; one that passes is enough.
+        if (verdicts[key] === undefined || result === "pass")
+            verdicts[key] = result;
+    }
+    return Object.keys(verdicts).length ? verdicts : undefined;
+};
+export const addressesOf = (value) => {
+    if (value === undefined)
+        return [];
+    const bracketed = [...value.matchAll(/<([^<>\s]+@[^<>\s]+)>/g)].map(([, address]) => address);
+    const found = bracketed.length ? bracketed : (value.match(/[^\s<>,;:"()]+@[^\s<>,;:"()]+/g) ?? []);
+    return found.map((address) => address.toLowerCase());
+};
+const nameOf = (value) => {
+    const before = /^\s*(.*?)\s*<[^<>]+>\s*$/.exec(value)?.[1];
+    const name = before?.replace(/^"(.*)"$/, "$1").trim();
+    return name ? decodeWords(name) : undefined;
+};
+const listIdOf = (raw) => raw === undefined ? undefined : (/<([^>]+)>/.exec(raw)?.[1] ?? raw).trim().toLowerCase() || undefined;
+/** The header fields of one email, from its header text. */
+export const headersOf = (text) => {
+    const unfolded = unfold(text);
+    const sender = field(unfolded, "From");
+    const [from] = addressesOf(sender);
+    const fromName = sender === undefined ? undefined : nameOf(sender);
+    const subject = field(unfolded, "Subject");
+    const listId = listIdOf(field(unfolded, "List-Id"));
+    const [replyTo] = addressesOf(field(unfolded, "Reply-To"));
+    const auth = authOf(unfolded);
+    const bulk = Boolean(listId) ||
+        field(unfolded, "List-Unsubscribe") !== undefined ||
+        /^(bulk|list|junk)\b/i.test(field(unfolded, "Precedence") ?? "");
+    return {
+        ...(from === undefined ? {} : { from }),
+        ...(fromName ? { fromName } : {}),
+        ...(subject === undefined ? {} : { subject: decodeWords(subject).trim() }),
+        to: [...addressesOf(field(unfolded, "To")), ...addressesOf(field(unfolded, "Cc"))],
+        ...(listId ? { listId } : {}),
+        bulk,
+        ...(replyTo === undefined ? {} : { replyTo }),
+        ...(auth === undefined ? {} : { auth }),
+        ...(/^\s*multipart\/mixed\b/i.test(field(unfolded, "Content-Type") ?? "") ? { attachments: true } : {}),
+    };
+};
+/**
+ * List-Id, the bulk-mail fields, Reply-To and Gmail's SPF, DKIM and DMARC verdicts of a whole email, kept on mail
+ * stored in full so a rule or a sender score can be worked out from the store later. Names are lower case.
+ */
+export const mailHeadersOf = (raw) => {
+    const bytes = Buffer.from(raw.buffer, raw.byteOffset, raw.byteLength);
+    const end = [bytes.indexOf("\r\n\r\n"), bytes.indexOf("\n\n")].filter((at) => at !== -1);
+    const head = bytes.subarray(0, end.length ? Math.min(...end) : Math.min(bytes.length, 256 * 1024));
+    const unfolded = unfold(head.toString("utf8"));
+    const listId = listIdOf(field(unfolded, "List-Id"));
+    const precedence = field(unfolded, "Precedence")?.trim().toLowerCase();
+    const [replyTo] = addressesOf(field(unfolded, "Reply-To"));
+    return {
+        ...(listId ? { "list-id": listId } : {}),
+        ...(replyTo === undefined ? {} : { "reply-to": replyTo }),
+        ...authOf(unfolded),
+        ...(field(unfolded, "List-Unsubscribe") === undefined ? {} : { "list-unsubscribe": true }),
+        ...(precedence ? { precedence } : {}),
+    };
+};

package/dist/mail/index.d.ts

@@ -0,0 +1,17 @@
+export { BULK, bulkBox } from "./bulk.js";
+export { factsOf } from "./facts.js";
+export { GMAIL_ALL_MAIL, mailFolders } from "./folders.js";
+export { dropMissing, type Gone, tooManyGone } from "./gone.js";
+export { type AuthVerdicts, HEADER_FIELDS, headersOf, type MailHeaders, mailHeadersOf } from "./headers.js";
+export { type Address, htmlToText, looksLikeHtml, MAX_BODY_CHARS, type ParsedMail, parseMail, } from "./parse.js";
+export { PRESET_NAMES, PRESETS, type PresetName, presetMatches } from "./presets.js";
+export { domainsToAge, groupKey, type Probe, type ProbeGroup, probeMail, SUGGEST_RULE } from "./probe.js";
+export { type FileStamp, FULL_PASS_MS, MBOX_RESUME, type MboxPoint, mergeRanges, parsePoint, type Ranges, type ResumePoint, rangesOf, resumeKey, rulesKey, within, } from "./resume.js";
+export { ruleInput, STORE_ACTION } from "./rule-input.js";
+export { mapProposedTags, ruleTagger, settleRuleTags, type TagOutcome } from "./rule-tags.js";
+export { conditionKeys, decide, describeMatch, ENVELOPE, envelopeBox, envelopeBoxOf, type MailAction, type MailFacts, type MailRule, type MailRuleMatch, matches, sameMatch, tagsOf, } from "./rules.js";
+export { type DomainAge, FRAUD_SIGNALS, FREE_MAIL, registrableOf, SCORE_RULE, type Scored, type Signal, scoreGroup, WEIGHTS, } from "./score.js";
+export { DEMOTE_AT, type DomainAges, demotedSenders, type LookupResult, SCORE_VERSION, scoreSenders, } from "./sender-scores.js";
+export type { Listed, MailSource, Searched } from "./source.js";
+export { listedOf, type StoredMail } from "./stored.js";
+export { MAX_ATTACHMENT_CHARS, type MailAccount, type MailIndexed, type MailSyncOptions, type MailSyncResult, RETRYABLE_ATTACHMENTS, syncMail, } from "./sync.js";

package/dist/mail/index.js

@@ -0,0 +1,16 @@
+export { BULK, bulkBox } from "./bulk.js";
+export { factsOf } from "./facts.js";
+export { GMAIL_ALL_MAIL, mailFolders } from "./folders.js";
+export { dropMissing, tooManyGone } from "./gone.js";
+export { HEADER_FIELDS, headersOf, mailHeadersOf } from "./headers.js";
+export { htmlToText, looksLikeHtml, MAX_BODY_CHARS, parseMail, } from "./parse.js";
+export { PRESET_NAMES, PRESETS, presetMatches } from "./presets.js";
+export { domainsToAge, groupKey, probeMail, SUGGEST_RULE } from "./probe.js";
+export { FULL_PASS_MS, MBOX_RESUME, mergeRanges, parsePoint, rangesOf, resumeKey, rulesKey, within, } from "./resume.js";
+export { ruleInput, STORE_ACTION } from "./rule-input.js";
+export { mapProposedTags, ruleTagger, settleRuleTags } from "./rule-tags.js";
+export { conditionKeys, decide, describeMatch, ENVELOPE, envelopeBox, envelopeBoxOf, matches, sameMatch, tagsOf, } from "./rules.js";
+export { FRAUD_SIGNALS, FREE_MAIL, registrableOf, SCORE_RULE, scoreGroup, WEIGHTS, } from "./score.js";
+export { DEMOTE_AT, demotedSenders, SCORE_VERSION, scoreSenders, } from "./sender-scores.js";
+export { listedOf } from "./stored.js";
+export { MAX_ATTACHMENT_CHARS, RETRYABLE_ATTACHMENTS, syncMail, } from "./sync.js";

package/dist/mail/parse.d.ts

@@ -0,0 +1,24 @@
+export interface Address {
+    email: string;
+    name: string | null;
+}
+export interface ParsedMail {
+    subject: string | null;
+    from: Address | null;
+    to: Address[];
+    cc: Address[];
+    bcc: Address[];
+    attachments: {
+        id: number;
+        name: string;
+        mime: string | null;
+        bytes: Uint8Array | null;
+    }[];
+    text: string;
+}
+export declare const MAX_BODY_CHARS = 1000000;
+export declare const htmlToText: (html: string) => string;
+/** Some senders put their HTML in the text/plain part; its first tag gives it away. */
+export declare const looksLikeHtml: (text: string) => boolean;
+/** A raw RFC 5322 message, as IMAP's `BODY[]` holds it. */
+export declare const parseMail: (raw: Uint8Array, maxBodyChars?: number) => Promise<ParsedMail>;

package/dist/mail/parse.js

@@ -0,0 +1,79 @@
+import { CliError } from "@wirecat/cli-base";
+import PostalMime, {} from "postal-mime";
+// The store cuts a body into pieces itself; this only guards against a pathological one, like a pasted log.
+export const MAX_BODY_CHARS = 1_000_000;
+const addresses = (list) => list
+    .flatMap((entry) => entry.group ?? [entry])
+    .flatMap(({ address, name }) => (address ? [{ email: address.toLowerCase(), name: name || null }] : []));
+const HIDDEN_OPEN = /<(script|style)/gi;
+const HIDDEN_CLOSE = { script: /<\/script>/gi, style: /<\/style>/gi };
+const ENTITIES = { nbsp: " ", amp: "&", lt: "<", gt: ">", quot: '"', "#39": "'" };
+// `<script[\s\S]*?</script>` as one regex rescans to the end from every unclosed opener; a tag found
+// unclosed once stays unclosed for every later opener, so each close is searched for at most once.
+const dropHidden = (html) => {
+    const unclosed = new Set();
+    let out = "";
+    let kept = 0;
+    for (const open of html.matchAll(HIDDEN_OPEN)) {
+        const name = open[1].toLowerCase();
+        if (open.index < kept || unclosed.has(name))
+            continue;
+        const close = HIDDEN_CLOSE[name];
+        close.lastIndex = open.index + open[0].length;
+        const end = close.exec(html);
+        if (!end) {
+            unclosed.add(name);
+            continue;
+        }
+        out += html.slice(kept, open.index);
+        kept = end.index + end[0].length;
+    }
+    return out + html.slice(kept);
+};
+// Past the last `>` no tag can close, yet `[^>]+` would rescan to the end from every `<` there: quadratic on `<<<<…`.
+const stripTags = (html) => {
+    const end = html.lastIndexOf(">") + 1;
+    return html.slice(0, end).replace(/<[^>]+>/g, "") + html.slice(end);
+};
+// Comments go first: a tag inside one, like a commented-out link, would leave its text and a stray `-->` behind.
+const dropComments = (html) => html.replace(/<!--[\s\S]*?(-->|$)/g, "");
+export const htmlToText = (html) => stripTags(dropHidden(dropComments(html)).replace(/<br\s*\/?>|<\/(p|div|li|tr|h\d)>/gi, "\n"))
+    // One pass, so `&amp;lt;` stays the text `&lt;` instead of being decoded twice into `<`.
+    .replace(/&(nbsp|amp|lt|gt|quot|#39);/g, (_entity, name) => ENTITIES[name])
+    // From the start of a run only: `[ \t]+\n` alone rescans a long run of spaces from each of them.
+    .replace(/(?<![ \t])[ \t]+\n/g, "\n")
+    .replace(/\n{3,}/g, "\n\n")
+    .trim();
+/** Some senders put their HTML in the text/plain part; its first tag gives it away. */
+export const looksLikeHtml = (text) => /^\s*<(!doctype\s+html|html|head|body|table|div|center|meta|style)\b/i.test(text);
+const bytesOf = (content) => typeof content === "string"
+    ? new TextEncoder().encode(content)
+    : content instanceof Uint8Array
+        ? content
+        : new Uint8Array(content);
+/** A raw RFC 5322 message, as IMAP's `BODY[]` holds it. */
+export const parseMail = async (raw, maxBodyChars = MAX_BODY_CHARS) => {
+    let email;
+    try {
+        email = await PostalMime.parse(raw);
+    }
+    catch (error) {
+        throw new CliError("invalid_response", `unreadable email: ${error instanceof Error ? error.message : error}`);
+    }
+    const plain = email.text?.trim() ?? "";
+    const text = plain && !looksLikeHtml(plain) ? plain : htmlToText(email.html ?? plain);
+    return {
+        subject: email.subject ?? null,
+        from: addresses(email.from === undefined ? [] : [email.from])[0] ?? null,
+        to: addresses(email.to ?? []),
+        cc: addresses(email.cc ?? []),
+        bcc: addresses(email.bcc ?? []),
+        attachments: email.attachments.slice(0, 50).map((part, id) => ({
+            id,
+            name: (part.filename ?? `attachment-${id}`).slice(0, 200),
+            mime: part.mimeType || null,
+            bytes: bytesOf(part.content),
+        })),
+        text: text.replace(/\r\n?/g, "\n").slice(0, maxBodyChars),
+    };
+};

package/dist/mail/presets.d.ts

@@ -0,0 +1,30 @@
+import type { MailFacts } from "./rules.js";
+/**
+ * Kinds of mail every mailbox gets, matched by who sends them and how, not by a list of the owner's senders. Mail
+ * from someone the owner wrote to never matches a preset, so a real conversation is never caught by one.
+ */
+export declare const PRESETS: {
+    readonly social: {
+        readonly about: "notifications from social networks";
+        readonly domains: readonly ["facebookmail.com", "facebook.com", "linkedin.com", "twitter.com", "x.com", "instagram.com", "vk.com", "ok.ru", "quora.com", "redditmail.com", "pinterest.com", "tiktok.com", "meetup.com", "tumblr.com", "youtube.com"];
+    };
+    readonly newsletters: {
+        readonly about: "mailing lists with an unsubscribe link: List-Id and List-Unsubscribe, or a newsletter service";
+        readonly domains: readonly ["substack.com", "mcsv.net", "mailchimpapp.net", "sendinblue.com", "beehiiv.com", "convertkit.com"];
+    };
+    readonly developer: {
+        readonly about: "code hosting, CI, package and error-tracker notifications";
+        readonly domains: readonly ["github.com", "gitlab.com", "bitbucket.org", "atlassian.net", "circleci.com", "travis-ci.com", "rollbar.com", "sentry.io", "vercel.com", "netlify.com", "npmjs.com", "docker.com", "heroku.com", "codecov.io", "libraries.io"];
+    };
+    readonly shopping: {
+        readonly about: "shops and promotions, and what Gmail files under Promotions";
+        readonly domains: readonly ["aliexpress.com", "aliexpress.ru", "ebay.com", "ozon.ru", "wildberries.ru", "steampowered.com", "booking.com", "zalando.com", "etsy.com", "ikea.com"];
+    };
+    readonly alerts: {
+        readonly about: "monitoring and uptime alerts";
+        readonly domains: readonly ["uptimerobot.com", "pingdom.com", "statuscake.com", "pagerduty.com", "opsgenie.com", "newrelic.com", "datadoghq.com", "betteruptime.com", "healthchecks.io"];
+    };
+};
+export type PresetName = keyof typeof PRESETS;
+export declare const PRESET_NAMES: PresetName[];
+export declare const presetMatches: (name: PresetName, facts: MailFacts) => boolean;

package/dist/mail/presets.js

@@ -0,0 +1,95 @@
+import { registrableOf } from "./score.js";
+/**
+ * Kinds of mail every mailbox gets, matched by who sends them and how, not by a list of the owner's senders. Mail
+ * from someone the owner wrote to never matches a preset, so a real conversation is never caught by one.
+ */
+export const PRESETS = {
+    social: {
+        about: "notifications from social networks",
+        domains: [
+            "facebookmail.com",
+            "facebook.com",
+            "linkedin.com",
+            "twitter.com",
+            "x.com",
+            "instagram.com",
+            "vk.com",
+            "ok.ru",
+            "quora.com",
+            "redditmail.com",
+            "pinterest.com",
+            "tiktok.com",
+            "meetup.com",
+            "tumblr.com",
+            "youtube.com",
+        ],
+    },
+    newsletters: {
+        about: "mailing lists with an unsubscribe link: List-Id and List-Unsubscribe, or a newsletter service",
+        domains: ["substack.com", "mcsv.net", "mailchimpapp.net", "sendinblue.com", "beehiiv.com", "convertkit.com"],
+    },
+    developer: {
+        about: "code hosting, CI, package and error-tracker notifications",
+        domains: [
+            "github.com",
+            "gitlab.com",
+            "bitbucket.org",
+            "atlassian.net",
+            "circleci.com",
+            "travis-ci.com",
+            "rollbar.com",
+            "sentry.io",
+            "vercel.com",
+            "netlify.com",
+            "npmjs.com",
+            "docker.com",
+            "heroku.com",
+            "codecov.io",
+            "libraries.io",
+        ],
+    },
+    shopping: {
+        about: "shops and promotions, and what Gmail files under Promotions",
+        domains: [
+            "aliexpress.com",
+            "aliexpress.ru",
+            "ebay.com",
+            "ozon.ru",
+            "wildberries.ru",
+            "steampowered.com",
+            "booking.com",
+            "zalando.com",
+            "etsy.com",
+            "ikea.com",
+        ],
+    },
+    alerts: {
+        about: "monitoring and uptime alerts",
+        domains: [
+            "uptimerobot.com",
+            "pingdom.com",
+            "statuscake.com",
+            "pagerduty.com",
+            "opsgenie.com",
+            "newrelic.com",
+            "datadoghq.com",
+            "betteruptime.com",
+            "healthchecks.io",
+        ],
+    },
+};
+export const PRESET_NAMES = Object.keys(PRESETS);
+export const presetMatches = (name, facts) => {
+    const { headers } = facts;
+    const sender = headers?.from;
+    if (sender === undefined || facts.replied)
+        return false;
+    const domain = registrableOf(sender);
+    if (PRESETS[name].domains.includes(domain))
+        return true;
+    if (name === "newsletters")
+        return headers?.listId !== undefined && headers.bulk;
+    if (name === "shopping")
+        return facts.labels?.includes("Promotions") ?? false;
+    return false;
+};

package/dist/mail/probe.d.ts

@@ -0,0 +1,49 @@
+import { type PresetName } from "./presets.js";
+import { type DomainAge, type Signal } from "./score.js";
+import type { Listed } from "./source.js";
+export declare const SUGGEST_RULE: string;
+export interface ProbeGroup {
+    /** The sender; for a list, the first sender seen. */
+    from: string | null;
+    listId: string | null;
+    emails: number;
+    bytes: number;
+    /** Share of emails not read; null when the source does not say. */
+    unread: number | null;
+    /** Share of emails marked as list or bulk mail. */
+    bulk: number;
+    firstYear: number;
+    lastYear: number;
+    /** The owner sent mail to one of the group's senders. */
+    replied: boolean;
+    suggest: "skip" | null;
+    /** How much the group looks like spam, a scam or a robot: the sum of its signals' weights (SCORE_RULE). */
+    score: number;
+    signals: Signal[];
+}
+export interface Probe {
+    emails: number;
+    bytes: number;
+    /** Mail the owner sent: not grouped, but its recipients tell who was replied to. */
+    sent: number;
+    /** Emails whose headers could not be read: not grouped. */
+    withoutHeaders: number;
+    groups: number;
+    items: ProbeGroup[];
+    /** How much mail each built-in preset would match: proposals the owner may turn into rules. */
+    presets: {
+        preset: PresetName;
+        emails: number;
+        groups: number;
+        bytes: number;
+    }[];
+}
+/** A list's mail is one group whoever sends it; other mail is grouped by sender. */
+export declare const groupKey: (headers: {
+    from?: string;
+    listId?: string;
+}) => string;
+/** Groups mail by List-Id when it has one, else by sender address; largest groups first. */
+export declare const probeMail: (items: Listed[], limit: number, ages?: ReadonlyMap<string, DomainAge>) => Probe;
+/** The sender domains of the groups worth asking a registry about: not a big mailbox provider, never replied to. */
+export declare const domainsToAge: (groups: ProbeGroup[], skip: ReadonlySet<string>) => string[];

package/dist/mail/probe.js

@@ -0,0 +1,81 @@
+import { PRESET_NAMES, presetMatches } from "./presets.js";
+import { registrableOf, scoreGroup } from "./score.js";
+export const SUGGEST_RULE = "suggest is skip when a group has at least 20 emails, at least 80% of them list or bulk mail " +
+    "(List-Id, List-Unsubscribe or Precedence: bulk) and the owner never sent mail to its sender";
+const share = (part, whole) => Math.round((part / whole) * 100) / 100;
+const isSent = (item) => item.labels?.includes("Sent") || /(^|\/)sent\b/i.test(item.folder ?? "");
+/** A list's mail is one group whoever sends it; other mail is grouped by sender. */
+export const groupKey = (headers) => headers.listId === undefined ? `from:${headers.from}` : `list:${headers.listId}`;
+/** Groups mail by List-Id when it has one, else by sender address; largest groups first. */
+export const probeMail = (items, limit, ages = new Map()) => {
+    const sent = items.filter(isSent);
+    const recipients = new Set(sent.flatMap((item) => item.headers?.to ?? []));
+    const groups = new Map();
+    let withoutHeaders = 0;
+    for (const item of items) {
+        if (isSent(item))
+            continue;
+        const headers = item.headers;
+        if (headers === undefined || (headers.from === undefined && headers.listId === undefined)) {
+            withoutHeaders++;
+            continue;
+        }
+        const key = groupKey(headers);
+        const group = groups.get(key);
+        if (group)
+            group.push(item);
+        else
+            groups.set(key, [item]);
+    }
+    const all = [...groups.values()].map((group) => {
+        const senders = new Set(group.flatMap((item) => (item.headers?.from === undefined ? [] : [item.headers.from])));
+        const known = group.filter((item) => item.seen !== undefined);
+        const years = group.map((item) => new Date(item.receivedAt).getUTCFullYear());
+        const bulk = share(group.filter((item) => item.headers?.bulk).length, group.length);
+        const replied = [...senders].some((sender) => recipients.has(sender));
+        return {
+            from: [...senders][0] ?? null,
+            listId: group[0]?.headers?.listId ?? null,
+            emails: group.length,
+            bytes: group.reduce((sum, item) => sum + (item.size ?? 0), 0),
+            unread: known.length ? share(known.filter((item) => !item.seen).length, known.length) : null,
+            bulk,
+            firstYear: years.reduce((a, b) => Math.min(a, b)),
+            lastYear: years.reduce((a, b) => Math.max(a, b)),
+            replied,
+            suggest: group.length >= 20 && bulk >= 0.8 && !replied ? "skip" : null,
+            ...scoreGroup(group, { replied, bulk, ages }),
+        };
+    });
+    const presets = PRESET_NAMES.map((preset) => {
+        const hit = [...groups.values()].filter((group) => {
+            const replied = group.some((item) => item.headers?.from !== undefined && recipients.has(item.headers.from));
+            return presetMatches(preset, { ...group[0], replied });
+        });
+        return {
+            preset,
+            emails: hit.reduce((sum, group) => sum + group.length, 0),
+            groups: hit.length,
+            bytes: hit.reduce((sum, group) => sum + group.reduce((total, item) => total + (item.size ?? 0), 0), 0),
+        };
+    }).filter(({ emails }) => emails > 0);
+    all.sort((a, b) => b.emails - a.emails || b.bytes - a.bytes);
+    return {
+        emails: items.length,
+        bytes: items.reduce((sum, item) => sum + (item.size ?? 0), 0),
+        sent: sent.length,
+        withoutHeaders,
+        groups: all.length,
+        items: all.slice(0, limit),
+        presets,
+    };
+};
+/** The sender domains of the groups worth asking a registry about: not a big mailbox provider, never replied to. */
+export const domainsToAge = (groups, skip) => [
+    ...new Set(groups.flatMap((group) => {
+        if (group.replied || group.from === null)
+            return [];
+        const domain = registrableOf(group.from);
+        return skip.has(domain) ? [] : [domain];
+    })),
+];

package/dist/mail/resume.d.ts

@@ -0,0 +1,61 @@
+import type { MailRule } from "./rules.js";
+/** Inclusive UID stretches, sorted and apart: a mailbox read newest first in batches stays a few stretches. */
+export type Ranges = [number, number][];
+/** Where a folder's listing stopped, kept in the store beside the mail it describes, so a restored backup agrees. */
+export interface ResumePoint {
+    validity: string;
+    done: Ranges;
+    /** The rules the done mail was decided under; other rules decide it again. */
+    rules: string;
+    /**
+     * The widest recent listing of every message, the only kind that proves deletions and refreshes labels and
+     * flags: a later run whose window it covers may list only what is not done.
+     */
+    full: {
+        since: string;
+        at: string;
+    };
+    /** The widest window searched: its messages, and how many of them are done; a narrower run leaves it. */
+    window: {
+        since: string;
+        total: number;
+        done: number;
+    };
+    /**
+     * The widest window ever listed, kept across a rule change, a new UIDVALIDITY or --rescan, which start `window`
+     * over: a timer's 30 days must never stand in for a mailbox read since 2004.
+     */
+    widest?: {
+        since: string;
+        total: number;
+        at: string;
+    };
+    /** What the latest run listed, whatever its window. */
+    last?: {
+        since: string;
+        total: number;
+        done: number;
+        at: string;
+    };
+    updatedAt: string;
+}
+/** A whole listing at least this often, so deletions, labels and read state are never more than a day old. */
+export declare const FULL_PASS_MS = 86400000;
+export declare const resumeKey: (folder: string) => string;
+/** An mbox file has no UIDs: an unchanged file read to the end has nothing new. */
+export declare const MBOX_RESUME = "mail_resume:mbox";
+export interface FileStamp {
+    size: number;
+    mtimeMs: number;
+}
+export interface MboxPoint extends FileStamp {
+    /** Every email of the file was read: a run cut short by --limit reads it again. */
+    complete: boolean;
+    updatedAt: string;
+}
+export declare const rangesOf: (values: Iterable<number>) => Ranges;
+export declare const mergeRanges: (a: Ranges, b: Ranges) => Ranges;
+export declare const within: (ranges: Ranges, value: number) => boolean;
+/** Only what decides mail: a changed reason or date is the same rule. */
+export declare const rulesKey: (rules: readonly MailRule[]) => string;
+export declare const parsePoint: <T>(value: string | undefined) => T | undefined;