@wirecat/cli-core 0.471.0 → 0.473.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/CHANGELOG.md +48 -0
  2. package/dist/cli/execution.d.ts +5 -0
  3. package/dist/cli/execution.js +5 -0
  4. package/dist/cli/help/first-screen.js +5 -0
  5. package/dist/cli/messenger/conversation-answers.d.ts +1 -0
  6. package/dist/cli/messenger/hearing-command.js +1 -1
  7. package/dist/cli/messenger/message-answers.d.ts +1 -0
  8. package/dist/cli/messenger/message-items.js +2 -0
  9. package/dist/cli/messenger/message-writes.d.ts +2 -0
  10. package/dist/cli/program.js +5 -3
  11. package/dist/entities/conversation-results.d.ts +2 -0
  12. package/dist/entities/evidence.d.ts +8 -0
  13. package/dist/entities/index.d.ts +19 -0
  14. package/dist/entities/knowledge-lists.d.ts +1 -0
  15. package/dist/entities/knowledge-lists.js +1 -0
  16. package/dist/entities/listener.d.ts +4 -0
  17. package/dist/entities/message-context.d.ts +3 -0
  18. package/dist/entities/message-write.d.ts +4 -0
  19. package/dist/entities/message.d.ts +1 -0
  20. package/dist/entities/message.js +1 -0
  21. package/dist/entities/store-records.d.ts +1 -0
  22. package/dist/errors/reasons.js +1 -0
  23. package/dist/mail/bulk.d.ts +7 -0
  24. package/dist/mail/bulk.js +3 -0
  25. package/dist/mail/facts.d.ts +9 -0
  26. package/dist/mail/facts.js +22 -0
  27. package/dist/mail/folders.d.ts +10 -0
  28. package/dist/mail/folders.js +13 -0
  29. package/dist/mail/gone.d.ts +20 -0
  30. package/dist/mail/gone.js +53 -0
  31. package/dist/mail/headers.d.ts +43 -0
  32. package/dist/mail/headers.js +88 -0
  33. package/dist/mail/index.d.ts +17 -0
  34. package/dist/mail/index.js +16 -0
  35. package/dist/mail/parse.d.ts +24 -0
  36. package/dist/mail/parse.js +79 -0
  37. package/dist/mail/presets.d.ts +30 -0
  38. package/dist/mail/presets.js +95 -0
  39. package/dist/mail/probe.d.ts +49 -0
  40. package/dist/mail/probe.js +81 -0
  41. package/dist/mail/resume.d.ts +61 -0
  42. package/dist/mail/resume.js +61 -0
  43. package/dist/mail/rule-input.d.ts +14 -0
  44. package/dist/mail/rule-input.js +32 -0
  45. package/dist/mail/rule-tags.d.ts +40 -0
  46. package/dist/mail/rule-tags.js +87 -0
  47. package/dist/mail/rules.d.ts +83 -0
  48. package/dist/mail/rules.js +108 -0
  49. package/dist/mail/score.d.ts +67 -0
  50. package/dist/mail/score.js +174 -0
  51. package/dist/mail/sender-scores.d.ts +30 -0
  52. package/dist/mail/sender-scores.js +96 -0
  53. package/dist/mail/source.d.ts +74 -0
  54. package/dist/mail/source.js +1 -0
  55. package/dist/mail/stored.d.ts +7 -0
  56. package/dist/mail/stored.js +36 -0
  57. package/dist/mail/sync.d.ts +106 -0
  58. package/dist/mail/sync.js +748 -0
  59. package/dist/mail/testing/contract.d.ts +20 -0
  60. package/dist/mail/testing/contract.js +106 -0
  61. package/dist/mail/testing/fake.d.ts +72 -0
  62. package/dist/mail/testing/fake.js +156 -0
  63. package/dist/mail/testing/index.d.ts +2 -0
  64. package/dist/mail/testing/index.js +2 -0
  65. package/dist/runs/error-detail.d.ts +2 -0
  66. package/dist/runs/error-detail.js +4 -0
  67. package/dist/speech/hearing.d.ts +2 -0
  68. package/dist/speech/hearing.js +10 -0
  69. package/dist/store/knowledge-query.js +3 -1
  70. package/dist/store/people.d.ts +2 -0
  71. package/dist/store/people.js +12 -0
  72. package/dist/store/sqlite/knowledge.d.ts +3 -1
  73. package/dist/store/sqlite/knowledge.js +7 -2
  74. package/dist/store/sqlite/task-keys.d.ts +1 -0
  75. package/dist/store/sqlite/task-keys.js +1 -0
  76. package/package.json +12 -1
@@ -0,0 +1,174 @@
1
+ import { isDisposableEmailDomain } from "disposable-email-domains-js";
2
+ import { getDomain } from "tldts";
3
+ /**
4
+ * How much each sign of spam, a scam or a robot adds to a sender's score. Gmail filters spam before it reaches
5
+ * All Mail, so these mostly find what got through: spoofed senders, scams and robots no rule covers yet.
6
+ */
7
+ export const WEIGHTS = {
8
+ spam_folder: 4,
9
+ dmarc_fail: 3,
10
+ brand_spoof: 3,
11
+ young_domain: 5,
12
+ reply_to_elsewhere: 1,
13
+ never_opened: 2,
14
+ dkim_not_passed: 2,
15
+ disposable_domain: 2,
16
+ bulk_never_replied: 1,
17
+ replied: -5,
18
+ };
19
+ /**
20
+ * The signs of a forged or fraudulent sender, as opposed to how the owner treats ordinary bulk mail. A failed DMARC
21
+ * check is one only beside another: alone it marks real senders with a broken setup or a forwarding path.
22
+ */
23
+ export const FRAUD_SIGNALS = ["dmarc_fail", "brand_spoof", "young_domain", "disposable_domain"];
24
+ export const SCORE_RULE = "score adds: in Gmail's Spam +4, Gmail's DMARC check failed +3 (only beside another sign of fraud), a brand in the display name sent from another " +
25
+ "domain +3, the domain under 180 days old when the mail came +5 (only with looked-up ages), Reply-To on another " +
26
+ "domain +1, none of its mail ever opened +2, Gmail's DKIM check not passed +2, a throwaway-mail domain +2, bulk mail " +
27
+ "never replied to +1, replied to by the owner -5";
28
+ /**
29
+ * DMARC failures before this date are not counted: senders then often published a policy their own mail did not yet
30
+ * meet.
31
+ */
32
+ export const DMARC_SINCE = "2022-01-01T00:00:00.000Z";
33
+ /** A domain registered too recently to trust: under this many days old when the mail arrived. */
34
+ export const YOUNG_DAYS = 180;
35
+ /** The part of a host a person registers: `mail.news.example.co.uk` is `example.co.uk`. */
36
+ export const registrableOf = (address) => {
37
+ const host = address.slice(address.lastIndexOf("@") + 1).toLowerCase();
38
+ return getDomain(host) ?? host;
39
+ };
40
+ /**
41
+ * The big mailbox providers. Their domains are years old and say nothing about the sender, so they are never
42
+ * looked up; a brand display name sent from one of them is the classic scam.
43
+ */
44
+ export const FREE_MAIL = new Set([
45
+ "gmail.com",
46
+ "googlemail.com",
47
+ "yahoo.com",
48
+ "outlook.com",
49
+ "hotmail.com",
50
+ "live.com",
51
+ "msn.com",
52
+ "icloud.com",
53
+ "me.com",
54
+ "aol.com",
55
+ "proton.me",
56
+ "protonmail.com",
57
+ "gmx.com",
58
+ "gmx.de",
59
+ "mail.com",
60
+ "yandex.ru",
61
+ "ya.ru",
62
+ "mail.ru",
63
+ "bk.ru",
64
+ "inbox.ru",
65
+ "list.ru",
66
+ "rambler.ru",
67
+ "zoho.com",
68
+ ]);
69
+ /** A brand and the first labels of the domains it really sends from. */
70
+ const BRANDS = {
71
+ paypal: ["paypal"],
72
+ apple: ["apple", "icloud"],
73
+ microsoft: [
74
+ "microsoft",
75
+ "outlook",
76
+ "office",
77
+ "office365",
78
+ "live",
79
+ "hotmail",
80
+ "azure",
81
+ "onedrive",
82
+ "microsoftadvertising",
83
+ ],
84
+ google: ["google", "gmail", "youtube", "googlemail"],
85
+ amazon: ["amazon", "amazonaws", "aws"],
86
+ netflix: ["netflix"],
87
+ facebook: ["facebook", "facebookmail", "meta", "fb"],
88
+ instagram: ["instagram", "facebookmail"],
89
+ whatsapp: ["whatsapp"],
90
+ linkedin: ["linkedin"],
91
+ dhl: ["dhl"],
92
+ fedex: ["fedex"],
93
+ docusign: ["docusign"],
94
+ dropbox: ["dropbox", "dropboxmail", "hellosign"],
95
+ binance: ["binance"],
96
+ coinbase: ["coinbase"],
97
+ revolut: ["revolut"],
98
+ };
99
+ /** Platforms that name courses, spaces and pages after other companies: "Google Cloud" from Coursera is a course. */
100
+ const NAMES_OTHERS = new Set(["coursera", "quora"]);
101
+ const named = (brand, text) => new RegExp(`\\b${brand}\\b`, "i").test(text);
102
+ /**
103
+ * The brand a display name claims while the address belongs to someone else, if any. "Google via LinkedIn" from
104
+ * LinkedIn's own domain is LinkedIn passing a page on; "PayPal via Desk" from anywhere else is still judged whole.
105
+ */
106
+ export const spoofedBrand = (name, address) => {
107
+ if (name === undefined || address === undefined)
108
+ return undefined;
109
+ const label = registrableOf(address).split(".")[0] ?? "";
110
+ if (NAMES_OTHERS.has(label))
111
+ return undefined;
112
+ const relay = /\s+via\s+(.+)$/i.exec(name)?.[1];
113
+ if (relay !== undefined &&
114
+ Object.entries(BRANDS).some(([brand, owned]) => named(brand, relay) && owned.includes(label)))
115
+ return undefined;
116
+ return Object.entries(BRANDS).find(([brand, owned]) => named(brand, name) && !owned.includes(label))?.[0];
117
+ };
118
+ const DAY_MS = 24 * 60 * 60 * 1000;
119
+ /**
120
+ * Under YOUNG_DAYS old at `at`; false when the age is not known, and when the registration came after `at`: the
121
+ * domain lapsed and someone registered it again later, which says nothing about the mail that came before.
122
+ */
123
+ export const youngAt = (age, at) => {
124
+ if (age?.created === undefined)
125
+ return false;
126
+ const old = Date.parse(at) - Date.parse(age.created);
127
+ return old >= 0 && old < YOUNG_DAYS * DAY_MS;
128
+ };
129
+ const most = (items, test) => items.filter((item) => test(item)).length * 2 >= items.length;
130
+ /**
131
+ * A sender group's score from the facts its emails carry. A sign counts when it holds for at least half the group's
132
+ * emails, so one odd email does not mark a sender; a spoofed brand counts on any one.
133
+ */
134
+ export const scoreGroup = (items, context) => {
135
+ const signals = [];
136
+ const judged = items.filter((item) => item.headers?.auth?.dmarc !== undefined && item.receivedAt >= DMARC_SINCE);
137
+ if (most(items, (item) => item.labels?.includes("Spam")))
138
+ signals.push("spam_folder");
139
+ // A failure with a valid DKIM signature is a sender signing for another domain, common before 2021, not a forgery;
140
+ // one that passes DMARC on any of its mail is real, and its failures came by a path that breaks it.
141
+ const forged = (item) => item.headers?.auth?.dmarc === "fail" && item.headers.auth.dkim !== "pass";
142
+ if (judged.length && most(judged, forged) && !judged.some((item) => item.headers?.auth?.dmarc === "pass"))
143
+ signals.push("dmarc_fail");
144
+ if (items.some((item) => spoofedBrand(item.headers?.fromName, item.headers?.from) !== undefined))
145
+ signals.push("brand_spoof");
146
+ const sender = items.find((item) => item.headers?.from !== undefined)?.headers?.from;
147
+ const domain = sender === undefined ? undefined : registrableOf(sender);
148
+ const first = items.reduce((a, b) => (a.receivedAt < b.receivedAt ? a : b));
149
+ if (domain !== undefined && youngAt(context.ages.get(domain), first.receivedAt))
150
+ signals.push("young_domain");
151
+ if (most(items, (item) => {
152
+ const { from, replyTo } = item.headers ?? {};
153
+ return from !== undefined && replyTo !== undefined && registrableOf(replyTo) !== registrableOf(from);
154
+ }))
155
+ signals.push("reply_to_elsewhere");
156
+ const known = items.filter((item) => item.seen !== undefined);
157
+ if (known.length && known.every((item) => !item.seen))
158
+ signals.push("never_opened");
159
+ // Gmail's stamp without a dkim= verdict means the mail carried no signature: that counts as not passed.
160
+ const stamped = items.filter((item) => item.headers?.auth !== undefined);
161
+ if (stamped.length && most(stamped, (item) => item.headers?.auth?.dkim !== "pass"))
162
+ signals.push("dkim_not_passed");
163
+ if (domain !== undefined && isDisposableEmailDomain(domain))
164
+ signals.push("disposable_domain");
165
+ if (context.bulk >= 0.8 && !context.replied)
166
+ signals.push("bulk_never_replied");
167
+ if (context.replied)
168
+ signals.push("replied");
169
+ const kept = signals.includes("dmarc_fail") &&
170
+ !signals.some((signal) => signal !== "dmarc_fail" && FRAUD_SIGNALS.includes(signal))
171
+ ? signals.filter((signal) => signal !== "dmarc_fail")
172
+ : signals;
173
+ return { score: kept.reduce((sum, signal) => sum + WEIGHTS[signal], 0), signals: kept };
174
+ };
@@ -0,0 +1,30 @@
1
+ import type { MessageStore } from "../store/index.js";
2
+ import { type MailRule } from "./rules.js";
3
+ import { type DomainAge } from "./score.js";
4
+ export interface LookupResult {
5
+ looked: number;
6
+ failed: number;
7
+ }
8
+ /** Registration dates of sender domains: the ones already known, and a lookup for the rest. */
9
+ export interface DomainAges {
10
+ known: () => ReadonlyMap<string, DomainAge>;
11
+ lookUp: (domains: readonly string[]) => Promise<LookupResult>;
12
+ }
13
+ /** Raised when the weights or the signals change, so every sender is scored again on the next sync. */
14
+ export declare const SCORE_VERSION = 4;
15
+ /** Signs of fraud adding up to this rank a sender's mail lower in search. */
16
+ export declare const DEMOTE_AT = 3;
17
+ /**
18
+ * Scores every sender of the account from the mail the store holds, as `memo mail inspect` scores a listing, and
19
+ * keeps the scores in the store, where search shows them. At most once a day, or at once after SCORE_VERSION
20
+ * changes. A sender in several groups (its own mail and a list) keeps the highest score. Domains not yet aged are
21
+ * looked up first, for senders heard from in the last two years.
22
+ */
23
+ export declare const scoreSenders: (store: MessageStore, address: string, ages: DomainAges, now?: number, rules?: readonly MailRule[]) => Promise<{
24
+ scored: number;
25
+ looked: number;
26
+ failed: number;
27
+ demoted: number;
28
+ } | undefined>;
29
+ /** Every mail account's senders whose mail search ranks lower, as the last scoring left them. */
30
+ export declare const demotedSenders: (store: MessageStore) => Promise<Set<string>>;
@@ -0,0 +1,96 @@
1
+ import { domainsToAge, groupKey, probeMail } from "./probe.js";
2
+ import { decide } from "./rules.js";
3
+ import { FRAUD_SIGNALS, FREE_MAIL, registrableOf, WEIGHTS } from "./score.js";
4
+ import { listedOf } from "./stored.js";
5
+ const SCORED = "mail_sender_scores";
6
+ /** Raised when the weights or the signals change, so every sender is scored again on the next sync. */
7
+ export const SCORE_VERSION = 4;
8
+ /** Signs of fraud adding up to this rank a sender's mail lower in search. */
9
+ export const DEMOTE_AT = 3;
10
+ const DAY = 86_400_000;
11
+ /** Only senders heard from this recently are looked up: an older sender's domain may have lapsed or changed hands. */
12
+ const RECENT = 2 * 365 * DAY;
13
+ /**
14
+ * Scores every sender of the account from the mail the store holds, as `memo mail inspect` scores a listing, and
15
+ * keeps the scores in the store, where search shows them. At most once a day, or at once after SCORE_VERSION
16
+ * changes. A sender in several groups (its own mail and a list) keeps the highest score. Domains not yet aged are
17
+ * looked up first, for senders heard from in the last two years.
18
+ */
19
+ export const scoreSenders = async (store, address, ages, now = Date.now(), rules = []) => {
20
+ const key = { provider: "email", account: address.toLowerCase() };
21
+ const last = (await store.sync.syncState(key, SCORED))?.value;
22
+ const done = last === undefined ? undefined : JSON.parse(last);
23
+ if (done !== undefined && done.version === SCORE_VERSION && now - done.at < DAY)
24
+ return undefined;
25
+ const stored = await store.accounts.storedAccount(key).catch(() => null);
26
+ if (stored === null)
27
+ return undefined;
28
+ const items = (await store.mail.emailFacts({ accountId: stored.id })).map((email) => {
29
+ const item = listedOf(email);
30
+ return email.outgoing && !item.labels?.includes("Sent")
31
+ ? { ...item, labels: [...(item.labels ?? []), "Sent"] }
32
+ : item;
33
+ });
34
+ // Each registry learns only its own domains' names; big mailbox providers and senders the owner wrote to are never asked.
35
+ const latest = new Map();
36
+ for (const item of items) {
37
+ if (item.headers === undefined)
38
+ continue;
39
+ const key = groupKey(item.headers);
40
+ if ((latest.get(key) ?? "") < item.receivedAt)
41
+ latest.set(key, item.receivedAt);
42
+ }
43
+ const since = new Date(now - RECENT).toISOString();
44
+ const recent = probeMail(items, Number.MAX_SAFE_INTEGER).items.filter((group) => (latest.get(groupKey({
45
+ ...(group.from === null ? {} : { from: group.from }),
46
+ ...(group.listId === null ? {} : { listId: group.listId }),
47
+ })) ?? "") >= since);
48
+ const lookups = await ages.lookUp(domainsToAge(recent, FREE_MAIL)).catch(() => ({ looked: 0, failed: 1 }));
49
+ const groups = new Map(probeMail(items, Number.MAX_SAFE_INTEGER, ages.known()).items.map((group) => [
50
+ groupKey({
51
+ ...(group.from === null ? {} : { from: group.from }),
52
+ ...(group.listId === null ? {} : { listId: group.listId }),
53
+ }),
54
+ group,
55
+ ]));
56
+ // The owner's own: a domain they wrote to anyone at, their own mailbox's domain, or a sender a protect rule keeps.
57
+ const trusted = new Set([address, ...items.filter((item) => item.labels?.includes("Sent")).flatMap((item) => item.headers?.to ?? [])]
58
+ .map((to) => registrableOf(to))
59
+ .filter((domain) => !FREE_MAIL.has(domain)));
60
+ const protectedSenders = new Set(items.filter((item) => decide(rules, item)?.protected).flatMap((item) => item.headers?.from?.toLowerCase() ?? []));
61
+ const fraud = (signals) => signals.reduce((sum, signal) => sum + (FRAUD_SIGNALS.includes(signal) ? WEIGHTS[signal] : 0), 0);
62
+ const scores = new Map();
63
+ for (const item of items) {
64
+ const from = item.headers?.from?.toLowerCase();
65
+ const group = item.headers === undefined ? undefined : groups.get(groupKey(item.headers));
66
+ if (from === undefined || group === undefined)
67
+ continue;
68
+ const held = scores.get(from);
69
+ if (held === undefined || group.score > held.score)
70
+ scores.set(from, {
71
+ address: from,
72
+ score: group.score,
73
+ signals: group.signals,
74
+ demoted: fraud(group.signals) >= DEMOTE_AT &&
75
+ !group.signals.includes("replied") &&
76
+ !trusted.has(registrableOf(from)) &&
77
+ !protectedSenders.has(from),
78
+ });
79
+ }
80
+ const demoted = [...scores.values()].filter((score) => score.demoted).map(({ address }) => address);
81
+ const written = await store.mail.setSenderScores(stored.id, [...scores.values()], now);
82
+ await store.sync.setSyncState(key, SCORED, JSON.stringify({ version: SCORE_VERSION, at: now, demoted }));
83
+ return { scored: written, ...lookups, demoted: demoted.length };
84
+ };
85
+ /** Every mail account's senders whose mail search ranks lower, as the last scoring left them. */
86
+ export const demotedSenders = async (store) => {
87
+ const senders = new Set();
88
+ for (const account of await store.accounts.accounts()) {
89
+ if (account.provider !== "email")
90
+ continue;
91
+ const value = (await store.sync.syncState(account, SCORED))?.value;
92
+ for (const sender of value === undefined ? [] : (JSON.parse(value).demoted ?? []))
93
+ senders.add(sender);
94
+ }
95
+ return senders;
96
+ };
@@ -0,0 +1,74 @@
1
+ import type { MailHeaders } from "./headers.js";
2
+ import type { FileStamp } from "./resume.js";
3
+ export interface Listed {
4
+ uid: string;
5
+ /** Gmail's own ids: the same in every folder, and they survive a UID reset. */
6
+ messageId: string;
7
+ threadId: string;
8
+ receivedAt: string;
9
+ folder?: string;
10
+ /** Gmail labels, plain names; absent when the answer could not be read, so nothing is changed. */
11
+ labels?: string[];
12
+ seen?: boolean;
13
+ flagged?: boolean;
14
+ /** RFC822.SIZE in bytes, to size fetch batches. */
15
+ size?: number;
16
+ /** Read only when asked for; absent when not asked or unreadable, so no rule applies to it. */
17
+ headers?: MailHeaders;
18
+ }
19
+ export interface Searched {
20
+ /** Absent when the server does not say; then no UID is trusted across runs. */
21
+ validity?: string;
22
+ uids: string[];
23
+ }
24
+ /** Everything the import asks of a mail client, so a transport can be replaced without touching the sync. */
25
+ export interface MailSource {
26
+ /**
27
+ * How many messages one `list` call reads in about one request's time, parallel connections included; a budgeted
28
+ * sync lists that many at a time, so a request of the budget is one call. Absent: 200.
29
+ */
30
+ listBatch?: number;
31
+ /** The same for `fetchRaw`. Absent: 50. */
32
+ fetchBatch?: number;
33
+ /**
34
+ * Every message in `folder` received since `since`; all or nothing, since deletions are proved from it.
35
+ * `headers` also reads From, To, Cc and the list fields, for the probe and the owner's rules.
36
+ */
37
+ list(since: Date, folder: string, options?: {
38
+ headers?: boolean;
39
+ searched?: Searched;
40
+ }): Promise<Listed[]>;
41
+ /**
42
+ * The folder's UIDs in the window and its UIDVALIDITY, without reading a message; `list` given them reads only
43
+ * those. Absent for a file, which has no UIDs.
44
+ */
45
+ search?(since: Date, folder: string): Promise<Searched>;
46
+ /** A file's size and modification time, so an unchanged file read to the end is not read again. */
47
+ stamp?(): Promise<FileStamp>;
48
+ /**
49
+ * The messages of `folder` with these Gmail ids, listed as `list` lists them; a message no longer there is left
50
+ * out. Absent where messages have no Gmail id: a file, or plain IMAP.
51
+ */
52
+ find?(messageIds: readonly string[], folder: string, options?: {
53
+ headers?: boolean;
54
+ }): Promise<Listed[]>;
55
+ /**
56
+ * Moves messages by Gmail id between folders on the server; ids `from` does not hold come back as missing.
57
+ * `known` gives each id's UID in `from`, already read, so none is looked up again.
58
+ */
59
+ move?(messageIds: readonly string[], from: string, to: string, known?: ReadonlyMap<string, string>): Promise<{
60
+ moved: string[];
61
+ missing: string[];
62
+ }>;
63
+ /** All Mail and Trash under the names this account's language gives them. */
64
+ folders?(): Promise<{
65
+ all: string;
66
+ trash: string;
67
+ }>;
68
+ /** Which of these UIDs `folder` still holds. */
69
+ present?(uids: readonly string[], folder: string): Promise<string[]>;
70
+ /** Puts labels back on messages in `folder`, by Gmail id. */
71
+ label?(messageIds: readonly string[], folder: string, labels: readonly string[]): Promise<void>;
72
+ /** Each item's raw RFC 5322 bytes, in order; one that could not be read is an Error, not a failed call. */
73
+ fetchRaw(items: Listed[]): Promise<(Uint8Array | Error)[]>;
74
+ }
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,7 @@
1
+ import type { EmailFacts } from "../store/index.js";
2
+ import type { Listed } from "./source.js";
3
+ export type StoredMail = Listed & {
4
+ envelope: boolean;
5
+ };
6
+ /** A stored email in the shape a Gmail listing has; it has no UID until Gmail is asked. */
7
+ export declare const listedOf: (email: EmailFacts) => StoredMail;
@@ -0,0 +1,36 @@
1
+ import { BULK } from "./bulk.js";
2
+ import { ENVELOPE } from "./rules.js";
3
+ /** A stored email in the shape a Gmail listing has; it has no UID until Gmail is asked. */
4
+ export const listedOf = (email) => {
5
+ const labels = email.mailboxes.filter(({ kind }) => kind === "label").map(({ name }) => name);
6
+ if (email.flagged && !labels.includes("Starred"))
7
+ labels.push("Starred");
8
+ return {
9
+ uid: "",
10
+ messageId: email.externalId,
11
+ threadId: "",
12
+ receivedAt: new Date(email.receivedAt ?? 0).toISOString(),
13
+ labels: labels.sort(),
14
+ seen: email.read,
15
+ flagged: email.flagged,
16
+ ...(email.size === null ? {} : { size: email.size }),
17
+ headers: {
18
+ ...(email.fromAddress === null ? {} : { from: email.fromAddress }),
19
+ ...(email.fromName === null ? {} : { fromName: email.fromName }),
20
+ ...(email.subject === null ? {} : { subject: email.subject }),
21
+ ...(email.listId === null ? {} : { listId: email.listId }),
22
+ to: email.recipients,
23
+ bulk: email.mailboxes.some(({ kind }) => kind === BULK),
24
+ ...(email.replyTo === null ? {} : { replyTo: email.replyTo }),
25
+ ...(email.dkim === null && email.dmarc === null
26
+ ? {}
27
+ : {
28
+ auth: {
29
+ ...(email.dkim === null ? {} : { dkim: email.dkim }),
30
+ ...(email.dmarc === null ? {} : { dmarc: email.dmarc }),
31
+ },
32
+ }),
33
+ },
34
+ envelope: email.mailboxes.some(({ kind }) => kind === ENVELOPE),
35
+ };
36
+ };
@@ -0,0 +1,106 @@
1
+ import type { BudgetStop, RunBudget } from "../background/budget.js";
2
+ import type { LoadEngine } from "../readers/types.js";
3
+ import type { AccountKey, MessageStore } from "../store/index.js";
4
+ import { type TagOutcome } from "./rule-tags.js";
5
+ import { type MailRule } from "./rules.js";
6
+ import type { DomainAge } from "./score.js";
7
+ import type { MailSource } from "./source.js";
8
+ export interface MailAccount {
9
+ /** The mailbox address: the store's account id. */
10
+ address: string;
11
+ mode?: "gmail" | "imap" | "mbox";
12
+ folders?: string[];
13
+ rules?: MailRule[];
14
+ }
15
+ /** What an index step reports, as memo's embedding does; the sync only passes it on. */
16
+ export interface MailIndexed {
17
+ chats: number;
18
+ chunks: number;
19
+ left: boolean;
20
+ notEmbedded?: unknown;
21
+ }
22
+ export interface MailSyncResult {
23
+ account: string;
24
+ since: string;
25
+ listed: number;
26
+ saved: number;
27
+ alreadyStored: number;
28
+ /** New emails a skip rule kept out: never read, never stored. */
29
+ skipped: number;
30
+ /** New emails an envelope rule recorded: sender, date, size and labels, never their text. */
31
+ envelopes: number;
32
+ deleted: number;
33
+ /** False when `max` stopped the run before every new message was read; the next run continues. */
34
+ complete: boolean;
35
+ /** `partial`: a budgeted run stopped before the whole window was read; the next run goes on. */
36
+ state?: "partial" | "complete";
37
+ /** Why a budgeted run stopped early: its request budget, its deadline, or a cancel. */
38
+ stopped?: BudgetStop;
39
+ /** Messages of the window this run did not get to read; a budgeted run only. */
40
+ remaining?: number;
41
+ /** Requests a budgeted run spent. */
42
+ requests?: number;
43
+ /** Emails that could not be read or parsed; they are tried again next run. */
44
+ unreadable?: {
45
+ message: string;
46
+ reason: string;
47
+ }[];
48
+ /** Why stored messages missing at the source were left alone, when they were. */
49
+ deletionsSkipped?: string;
50
+ /** "resumed": only mail not done before was listed; "full": every message in the window was. */
51
+ pass?: "full" | "resumed";
52
+ indexed?: MailIndexed;
53
+ coverage?: {
54
+ folders: string[];
55
+ since: string;
56
+ until: string;
57
+ state: "window" | "partial";
58
+ identity: string;
59
+ };
60
+ attachments?: {
61
+ message: string;
62
+ part: number;
63
+ status: string;
64
+ }[];
65
+ /** Rule tags put on mail as an existing label the store pointed to. */
66
+ tagsReused?: TagOutcome["reused"];
67
+ /** Rule tags the store refused with nothing to reuse; that mail is stored untagged. */
68
+ tagsSkipped?: TagOutcome["skipped"];
69
+ /** `--retry-attachments`: emails read again for attachments a missing reader left unread. */
70
+ retried?: {
71
+ emails: number;
72
+ extracted: number;
73
+ stillFailed: number;
74
+ missing: number;
75
+ };
76
+ }
77
+ export declare const MAX_ATTACHMENT_CHARS = 200000;
78
+ export interface MailSyncOptions {
79
+ account: MailAccount;
80
+ since: Date;
81
+ max?: number;
82
+ maxBodyChars?: number;
83
+ maxAttachmentChars?: number;
84
+ now?: () => number;
85
+ /** Domain registration dates already known, for the sender score a rule asks for. */
86
+ ages?: ReadonlyMap<string, DomainAge>;
87
+ loadEngine?: LoadEngine;
88
+ retryAttachments?: boolean;
89
+ /** List every message in the window, not only those an earlier run left: proves deletions, refreshes labels. */
90
+ rescan?: boolean;
91
+ /** A window smaller than this is listed whole every run: it costs seconds and keeps flags and deletions current. */
92
+ resumeFrom?: number;
93
+ /** Runs after the mail is saved, so chats and mail still waiting for vectors are indexed; its answer is `indexed`. */
94
+ index?: (store: MessageStore, key: AccountKey) => Promise<MailIndexed>;
95
+ /** Stops between batches of emails, after saving the resume point. */
96
+ signal?: AbortSignal;
97
+ /**
98
+ * Makes the run budgeted: it reads the window in rounds, one request of the budget per call to the source, saves
99
+ * after each, stops when the budget says so and answers `state: "partial"`. A message the source could not give
100
+ * is tried once more at the end of the run and, if it still fails, left for the next one. `max` defaults to no limit.
101
+ */
102
+ budget?: RunBudget;
103
+ }
104
+ export declare const syncMail: (source: MailSource, store: MessageStore, options: MailSyncOptions) => Promise<MailSyncResult>;
105
+ /** Failures a newer install mends: the reader was missing, not the file at fault, so reading again can succeed. */
106
+ export declare const RETRYABLE_ATTACHMENTS: string[];