@wirecat/cli-core 0.470.0 → 0.472.0

package/dist/mail/sender-scores.d.ts

@@ -0,0 +1,30 @@
+import type { MessageStore } from "../store/index.js";
+import { type MailRule } from "./rules.js";
+import { type DomainAge } from "./score.js";
+export interface LookupResult {
+    looked: number;
+    failed: number;
+}
+/** Registration dates of sender domains: the ones already known, and a lookup for the rest. */
+export interface DomainAges {
+    known: () => ReadonlyMap<string, DomainAge>;
+    lookUp: (domains: readonly string[]) => Promise<LookupResult>;
+}
+/** Raised when the weights or the signals change, so every sender is scored again on the next sync. */
+export declare const SCORE_VERSION = 4;
+/** Signs of fraud adding up to this rank a sender's mail lower in search. */
+export declare const DEMOTE_AT = 3;
+/**
+ * Scores every sender of the account from the mail the store holds, as `memo mail inspect` scores a listing, and
+ * keeps the scores in the store, where search shows them. At most once a day, or at once after SCORE_VERSION
+ * changes. A sender in several groups (its own mail and a list) keeps the highest score. Domains not yet aged are
+ * looked up first, for senders heard from in the last two years.
+ */
+export declare const scoreSenders: (store: MessageStore, address: string, ages: DomainAges, now?: number, rules?: readonly MailRule[]) => Promise<{
+    scored: number;
+    looked: number;
+    failed: number;
+    demoted: number;
+} | undefined>;
+/** Every mail account's senders whose mail search ranks lower, as the last scoring left them. */
+export declare const demotedSenders: (store: MessageStore) => Promise<Set<string>>;

package/dist/mail/sender-scores.js

@@ -0,0 +1,96 @@
+import { domainsToAge, groupKey, probeMail } from "./probe.js";
+import { decide } from "./rules.js";
+import { FRAUD_SIGNALS, FREE_MAIL, registrableOf, WEIGHTS } from "./score.js";
+import { listedOf } from "./stored.js";
+const SCORED = "mail_sender_scores";
+/** Raised when the weights or the signals change, so every sender is scored again on the next sync. */
+export const SCORE_VERSION = 4;
+/** Signs of fraud adding up to this rank a sender's mail lower in search. */
+export const DEMOTE_AT = 3;
+const DAY = 86_400_000;
+/** Only senders heard from this recently are looked up: an older sender's domain may have lapsed or changed hands. */
+const RECENT = 2 * 365 * DAY;
+/**
+ * Scores every sender of the account from the mail the store holds, as `memo mail inspect` scores a listing, and
+ * keeps the scores in the store, where search shows them. At most once a day, or at once after SCORE_VERSION
+ * changes. A sender in several groups (its own mail and a list) keeps the highest score. Domains not yet aged are
+ * looked up first, for senders heard from in the last two years.
+ */
+export const scoreSenders = async (store, address, ages, now = Date.now(), rules = []) => {
+    const key = { provider: "email", account: address.toLowerCase() };
+    const last = (await store.sync.syncState(key, SCORED))?.value;
+    const done = last === undefined ? undefined : JSON.parse(last);
+    if (done !== undefined && done.version === SCORE_VERSION && now - done.at < DAY)
+        return undefined;
+    const stored = await store.accounts.storedAccount(key).catch(() => null);
+    if (stored === null)
+        return undefined;
+    const items = (await store.mail.emailFacts({ accountId: stored.id })).map((email) => {
+        const item = listedOf(email);
+        return email.outgoing && !item.labels?.includes("Sent")
+            ? { ...item, labels: [...(item.labels ?? []), "Sent"] }
+            : item;
+    });
+    // Each registry learns only its own domains' names; big mailbox providers and senders the owner wrote to are never asked.
+    const latest = new Map();
+    for (const item of items) {
+        if (item.headers === undefined)
+            continue;
+        const key = groupKey(item.headers);
+        if ((latest.get(key) ?? "") < item.receivedAt)
+            latest.set(key, item.receivedAt);
+    }
+    const since = new Date(now - RECENT).toISOString();
+    const recent = probeMail(items, Number.MAX_SAFE_INTEGER).items.filter((group) => (latest.get(groupKey({
+        ...(group.from === null ? {} : { from: group.from }),
+        ...(group.listId === null ? {} : { listId: group.listId }),
+    })) ?? "") >= since);
+    const lookups = await ages.lookUp(domainsToAge(recent, FREE_MAIL)).catch(() => ({ looked: 0, failed: 1 }));
+    const groups = new Map(probeMail(items, Number.MAX_SAFE_INTEGER, ages.known()).items.map((group) => [
+        groupKey({
+            ...(group.from === null ? {} : { from: group.from }),
+            ...(group.listId === null ? {} : { listId: group.listId }),
+        }),
+        group,
+    ]));
+    // The owner's own: a domain they wrote to anyone at, their own mailbox's domain, or a sender a protect rule keeps.
+    const trusted = new Set([address, ...items.filter((item) => item.labels?.includes("Sent")).flatMap((item) => item.headers?.to ?? [])]
+        .map((to) => registrableOf(to))
+        .filter((domain) => !FREE_MAIL.has(domain)));
+    const protectedSenders = new Set(items.filter((item) => decide(rules, item)?.protected).flatMap((item) => item.headers?.from?.toLowerCase() ?? []));
+    const fraud = (signals) => signals.reduce((sum, signal) => sum + (FRAUD_SIGNALS.includes(signal) ? WEIGHTS[signal] : 0), 0);
+    const scores = new Map();
+    for (const item of items) {
+        const from = item.headers?.from?.toLowerCase();
+        const group = item.headers === undefined ? undefined : groups.get(groupKey(item.headers));
+        if (from === undefined || group === undefined)
+            continue;
+        const held = scores.get(from);
+        if (held === undefined || group.score > held.score)
+            scores.set(from, {
+                address: from,
+                score: group.score,
+                signals: group.signals,
+                demoted: fraud(group.signals) >= DEMOTE_AT &&
+                    !group.signals.includes("replied") &&
+                    !trusted.has(registrableOf(from)) &&
+                    !protectedSenders.has(from),
+            });
+    }
+    const demoted = [...scores.values()].filter((score) => score.demoted).map(({ address }) => address);
+    const written = await store.mail.setSenderScores(stored.id, [...scores.values()], now);
+    await store.sync.setSyncState(key, SCORED, JSON.stringify({ version: SCORE_VERSION, at: now, demoted }));
+    return { scored: written, ...lookups, demoted: demoted.length };
+};
+/** Every mail account's senders whose mail search ranks lower, as the last scoring left them. */
+export const demotedSenders = async (store) => {
+    const senders = new Set();
+    for (const account of await store.accounts.accounts()) {
+        if (account.provider !== "email")
+            continue;
+        const value = (await store.sync.syncState(account, SCORED))?.value;
+        for (const sender of value === undefined ? [] : (JSON.parse(value).demoted ?? []))
+            senders.add(sender);
+    }
+    return senders;
+};

package/dist/mail/source.d.ts

@@ -0,0 +1,67 @@
+import type { MailHeaders } from "./headers.js";
+import type { FileStamp } from "./resume.js";
+export interface Listed {
+    uid: string;
+    /** Gmail's own ids: the same in every folder, and they survive a UID reset. */
+    messageId: string;
+    threadId: string;
+    receivedAt: string;
+    folder?: string;
+    /** Gmail labels, plain names; absent when the answer could not be read, so nothing is changed. */
+    labels?: string[];
+    seen?: boolean;
+    flagged?: boolean;
+    /** RFC822.SIZE in bytes, to size fetch batches. */
+    size?: number;
+    /** Read only when asked for; absent when not asked or unreadable, so no rule applies to it. */
+    headers?: MailHeaders;
+}
+export interface Searched {
+    /** Absent when the server does not say; then no UID is trusted across runs. */
+    validity?: string;
+    uids: string[];
+}
+/** Everything the import asks of a mail client, so a transport can be replaced without touching the sync. */
+export interface MailSource {
+    /**
+     * Every message in `folder` received since `since`; all or nothing, since deletions are proved from it.
+     * `headers` also reads From, To, Cc and the list fields, for the probe and the owner's rules.
+     */
+    list(since: Date, folder: string, options?: {
+        headers?: boolean;
+        searched?: Searched;
+    }): Promise<Listed[]>;
+    /**
+     * The folder's UIDs in the window and its UIDVALIDITY, without reading a message; `list` given them reads only
+     * those. Absent for a file, which has no UIDs.
+     */
+    search?(since: Date, folder: string): Promise<Searched>;
+    /** A file's size and modification time, so an unchanged file read to the end is not read again. */
+    stamp?(): Promise<FileStamp>;
+    /**
+     * The messages of `folder` with these Gmail ids, listed as `list` lists them; a message no longer there is left
+     * out. Absent where messages have no Gmail id: a file, or plain IMAP.
+     */
+    find?(messageIds: readonly string[], folder: string, options?: {
+        headers?: boolean;
+    }): Promise<Listed[]>;
+    /**
+     * Moves messages by Gmail id between folders on the server; ids `from` does not hold come back as missing.
+     * `known` gives each id's UID in `from`, already read, so none is looked up again.
+     */
+    move?(messageIds: readonly string[], from: string, to: string, known?: ReadonlyMap<string, string>): Promise<{
+        moved: string[];
+        missing: string[];
+    }>;
+    /** All Mail and Trash under the names this account's language gives them. */
+    folders?(): Promise<{
+        all: string;
+        trash: string;
+    }>;
+    /** Which of these UIDs `folder` still holds. */
+    present?(uids: readonly string[], folder: string): Promise<string[]>;
+    /** Puts labels back on messages in `folder`, by Gmail id. */
+    label?(messageIds: readonly string[], folder: string, labels: readonly string[]): Promise<void>;
+    /** Each item's raw RFC 5322 bytes, in order; one that could not be read is an Error, not a failed call. */
+    fetchRaw(items: Listed[]): Promise<(Uint8Array | Error)[]>;
+}

package/dist/mail/source.js

@@ -0,0 +1 @@
+export {};

package/dist/mail/stored.d.ts

@@ -0,0 +1,7 @@
+import type { EmailFacts } from "../store/index.js";
+import type { Listed } from "./source.js";
+export type StoredMail = Listed & {
+    envelope: boolean;
+};
+/** A stored email in the shape a Gmail listing has; it has no UID until Gmail is asked. */
+export declare const listedOf: (email: EmailFacts) => StoredMail;

package/dist/mail/stored.js

@@ -0,0 +1,36 @@
+import { BULK } from "./bulk.js";
+import { ENVELOPE } from "./rules.js";
+/** A stored email in the shape a Gmail listing has; it has no UID until Gmail is asked. */
+export const listedOf = (email) => {
+    const labels = email.mailboxes.filter(({ kind }) => kind === "label").map(({ name }) => name);
+    if (email.flagged && !labels.includes("Starred"))
+        labels.push("Starred");
+    return {
+        uid: "",
+        messageId: email.externalId,
+        threadId: "",
+        receivedAt: new Date(email.receivedAt ?? 0).toISOString(),
+        labels: labels.sort(),
+        seen: email.read,
+        flagged: email.flagged,
+        ...(email.size === null ? {} : { size: email.size }),
+        headers: {
+            ...(email.fromAddress === null ? {} : { from: email.fromAddress }),
+            ...(email.fromName === null ? {} : { fromName: email.fromName }),
+            ...(email.subject === null ? {} : { subject: email.subject }),
+            ...(email.listId === null ? {} : { listId: email.listId }),
+            to: email.recipients,
+            bulk: email.mailboxes.some(({ kind }) => kind === BULK),
+            ...(email.replyTo === null ? {} : { replyTo: email.replyTo }),
+            ...(email.dkim === null && email.dmarc === null
+                ? {}
+                : {
+                    auth: {
+                        ...(email.dkim === null ? {} : { dkim: email.dkim }),
+                        ...(email.dmarc === null ? {} : { dmarc: email.dmarc }),
+                    },
+                }),
+        },
+        envelope: email.mailboxes.some(({ kind }) => kind === ENVELOPE),
+    };
+};

package/dist/mail/sync.d.ts

@@ -0,0 +1,91 @@
+import type { LoadEngine } from "../readers/types.js";
+import type { AccountKey, MessageStore } from "../store/index.js";
+import { type TagOutcome } from "./rule-tags.js";
+import { type MailRule } from "./rules.js";
+import type { DomainAge } from "./score.js";
+import type { MailSource } from "./source.js";
+export interface MailAccount {
+    /** The mailbox address: the store's account id. */
+    address: string;
+    mode?: "gmail" | "imap" | "mbox";
+    folders?: string[];
+    rules?: MailRule[];
+}
+/** What an index step reports, as memo's embedding does; the sync only passes it on. */
+export interface MailIndexed {
+    chats: number;
+    chunks: number;
+    left: boolean;
+    notEmbedded?: unknown;
+}
+export interface MailSyncResult {
+    account: string;
+    since: string;
+    listed: number;
+    saved: number;
+    alreadyStored: number;
+    /** New emails a skip rule kept out: never read, never stored. */
+    skipped: number;
+    /** New emails an envelope rule recorded: sender, date, size and labels, never their text. */
+    envelopes: number;
+    deleted: number;
+    /** False when `max` stopped the run before every new message was read; the next run continues. */
+    complete: boolean;
+    /** Emails that could not be read or parsed; they are tried again next run. */
+    unreadable?: {
+        message: string;
+        reason: string;
+    }[];
+    /** Why stored messages missing at the source were left alone, when they were. */
+    deletionsSkipped?: string;
+    /** "resumed": only mail not done before was listed; "full": every message in the window was. */
+    pass?: "full" | "resumed";
+    indexed?: MailIndexed;
+    coverage?: {
+        folders: string[];
+        since: string;
+        until: string;
+        state: "window" | "partial";
+        identity: string;
+    };
+    attachments?: {
+        message: string;
+        part: number;
+        status: string;
+    }[];
+    /** Rule tags put on mail as an existing label the store pointed to. */
+    tagsReused?: TagOutcome["reused"];
+    /** Rule tags the store refused with nothing to reuse; that mail is stored untagged. */
+    tagsSkipped?: TagOutcome["skipped"];
+    /** `--retry-attachments`: emails read again for attachments a missing reader left unread. */
+    retried?: {
+        emails: number;
+        extracted: number;
+        stillFailed: number;
+        missing: number;
+    };
+}
+export declare const MAX_ATTACHMENT_CHARS = 200000;
+export interface MailSyncOptions {
+    account: MailAccount;
+    since: Date;
+    max?: number;
+    maxBodyChars?: number;
+    maxAttachmentChars?: number;
+    now?: () => number;
+    /** Domain registration dates already known, for the sender score a rule asks for. */
+    ages?: ReadonlyMap<string, DomainAge>;
+    loadEngine?: LoadEngine;
+    retryAttachments?: boolean;
+    /** List every message in the window, not only those an earlier run left: proves deletions, refreshes labels. */
+    rescan?: boolean;
+    /** A window smaller than this is listed whole every run: it costs seconds and keeps flags and deletions current. */
+    resumeFrom?: number;
+    /** Runs after the mail is saved, so chats and mail still waiting for vectors are indexed; its answer is `indexed`. */
+    index?: (store: MessageStore, key: AccountKey) => Promise<MailIndexed>;
+    /** Stops between batches of emails, after saving the resume point. */
+    signal?: AbortSignal;
+}
+export declare const syncMail: (source: MailSource, store: MessageStore, { account, since, max, maxBodyChars, maxAttachmentChars, now, ages, loadEngine, retryAttachments, rescan, resumeFrom, index, signal, }: MailSyncOptions) => Promise<MailSyncResult>;
+/** Failures a newer install mends: the reader was missing, not the file at fault, so reading again can succeed. */
+export declare const RETRYABLE_ATTACHMENTS: string[];