@wirecat/cli-core 0.486.0 → 0.487.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/dist/cli/messenger/context.d.ts +2 -0
- package/dist/cli/messenger/reactions-command.js +4 -0
- package/dist/cli/messenger/serve-relay.d.ts +1 -0
- package/dist/mail/headers.d.ts +2 -0
- package/dist/mail/headers.js +8 -4
- package/dist/mail/index.d.ts +1 -1
- package/dist/mail/quick.d.ts +39 -0
- package/dist/mail/quick.js +73 -0
- package/dist/mail/source.d.ts +20 -1
- package/dist/mail/sync.js +55 -6
- package/dist/mail/testing/contract.js +22 -0
- package/dist/mail/testing/fake.d.ts +5 -0
- package/dist/mail/testing/fake.js +28 -5
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -7,6 +7,21 @@ shape of this file.
|
|
|
7
7
|
|
|
8
8
|
## Unreleased
|
|
9
9
|
|
|
10
|
+
## 0.487.0 — 30.10.2026
|
|
11
|
+
|
|
12
|
+
### Added
|
|
13
|
+
|
|
14
|
+
- **A messenger can say what `reactions list` covers.** The new `reactionsListScope` option on the messenger (for
|
|
15
|
+
WhatsApp, "reactions received while this computer was syncing") adds `missing: [{ field: "reactions", why }]` to the
|
|
16
|
+
JSON answer and one line on stderr for a person: "shows only …". A messenger that sets none answers as before.
|
|
17
|
+
- **Mail from a server with no cheap message id (Yandex) syncs from a listing of UID, flags, date and size.** A header
|
|
18
|
+
FETCH of 200 messages took 5 to 40 s there and often failed, so six months took hours. A `MailSource` can now answer
|
|
19
|
+
`quick: true` from `search` and give `quick` and `identify`; `syncMail` then lists only the UIDs not done, reads the
|
|
20
|
+
raw bytes of the new ones and takes the id from the Message-ID, so stored mail keeps its id and nothing stored is read
|
|
21
|
+
again. Rules and the bulk class run on the downloaded headers. A new UIDVALIDITY, other rules or `rescan` list the
|
|
22
|
+
folder with headers as before; a quick run proves no deletion and refreshes no flags of the mail it skips.
|
|
23
|
+
`mailSourceContract` checks that `quick` and `identify` agree with `list`; `fakeMailSource` takes `quick: true`.
|
|
24
|
+
|
|
10
25
|
## 0.486.0 — 30.10.2026
|
|
11
26
|
|
|
12
27
|
### Fixed
|
|
@@ -84,6 +84,8 @@ export interface Messenger extends ServiceMessenger {
|
|
|
84
84
|
pollCloseSeconds?: readonly [min: number, max: number];
|
|
85
85
|
/** Whether `reactions list` shows who reacted to a message; no when unset. */
|
|
86
86
|
reactionsList?: boolean;
|
|
87
|
+
/** What `reactions list` covers when it is less than everyone who reacted, as the end of "shows only …", such as "reactions received while this computer was syncing"; none when unset. */
|
|
88
|
+
reactionsListScope?: string;
|
|
87
89
|
/** Whether `polls voters` lists who voted for what; no when unset. */
|
|
88
90
|
pollVoters?: boolean;
|
|
89
91
|
/** Whether a poll can be closed — `polls close`; yes when unset. WhatsApp's stay open. */
|
|
@@ -33,6 +33,7 @@ export const reactionsCommand = (messenger) => {
|
|
|
33
33
|
...(cursor === undefined ? {} : { cursor }),
|
|
34
34
|
});
|
|
35
35
|
});
|
|
36
|
+
const why = messenger.reactionsListScope === undefined ? undefined : `shows only ${messenger.reactionsListScope}`;
|
|
36
37
|
renderAnswer(this, context, listOf("reactions list", {
|
|
37
38
|
items: found.items.map(reactorAnswer),
|
|
38
39
|
limit,
|
|
@@ -40,7 +41,10 @@ export const reactionsCommand = (messenger) => {
|
|
|
40
41
|
paging: "cursor",
|
|
41
42
|
...(found.cursor === undefined ? {} : { cursor: found.cursor }),
|
|
42
43
|
args: { chat, message, ...(words.length > 0 ? { query: words.join(" ") } : {}) },
|
|
44
|
+
...(why === undefined ? {} : { missing: [{ field: "reactions", why }] }),
|
|
43
45
|
}), lines(({ reacted_at, person, reaction: name }) => [reacted_at, person.name ?? person.id, name].join(" ")));
|
|
46
|
+
if (why !== undefined && context.format !== "json")
|
|
47
|
+
context.renderer.note(why);
|
|
44
48
|
}));
|
|
45
49
|
annotate(reactions.command("add"), {
|
|
46
50
|
effects: REMOTE_WRITE,
|
|
@@ -45,6 +45,7 @@ export declare const serveRelay: (command: Command, messenger: Messenger) => {
|
|
|
45
45
|
pollQuiz?: boolean;
|
|
46
46
|
pollCloseSeconds?: readonly [min: number, max: number];
|
|
47
47
|
reactionsList?: boolean;
|
|
48
|
+
reactionsListScope?: string;
|
|
48
49
|
pollVoters?: boolean;
|
|
49
50
|
pollClose?: boolean;
|
|
50
51
|
pollRevote?: boolean;
|
package/dist/mail/headers.d.ts
CHANGED
|
@@ -36,6 +36,8 @@ export declare const authOf: (unfolded: string) => AuthVerdicts | undefined;
|
|
|
36
36
|
export declare const addressesOf: (value: string | undefined) => string[];
|
|
37
37
|
/** The header fields of one email, from its header text. */
|
|
38
38
|
export declare const headersOf: (text: string) => MailHeaders;
|
|
39
|
+
/** The header fields of a whole email, as `headersOf` reads them from a listing. */
|
|
40
|
+
export declare const headersOfRaw: (raw: Uint8Array) => MailHeaders;
|
|
39
41
|
/**
|
|
40
42
|
* List-Id, the bulk-mail fields, Reply-To and Gmail's SPF, DKIM and DMARC verdicts of a whole email, kept on mail
|
|
41
43
|
* stored in full so a rule or a sender score can be worked out from the store later. Names are lower case.
|
package/dist/mail/headers.js
CHANGED
|
@@ -66,15 +66,19 @@ export const headersOf = (text) => {
|
|
|
66
66
|
...(/^\s*multipart\/mixed\b/i.test(field(unfolded, "Content-Type") ?? "") ? { attachments: true } : {}),
|
|
67
67
|
};
|
|
68
68
|
};
|
|
69
|
+
const headTextOf = (raw) => {
|
|
70
|
+
const bytes = Buffer.from(raw.buffer, raw.byteOffset, raw.byteLength);
|
|
71
|
+
const end = [bytes.indexOf("\r\n\r\n"), bytes.indexOf("\n\n")].filter((at) => at !== -1);
|
|
72
|
+
return bytes.subarray(0, end.length ? Math.min(...end) : Math.min(bytes.length, 256 * 1024)).toString("utf8");
|
|
73
|
+
};
|
|
74
|
+
/** The header fields of a whole email, as `headersOf` reads them from a listing. */
|
|
75
|
+
export const headersOfRaw = (raw) => headersOf(headTextOf(raw));
|
|
69
76
|
/**
|
|
70
77
|
* List-Id, the bulk-mail fields, Reply-To and Gmail's SPF, DKIM and DMARC verdicts of a whole email, kept on mail
|
|
71
78
|
* stored in full so a rule or a sender score can be worked out from the store later. Names are lower case.
|
|
72
79
|
*/
|
|
73
80
|
export const mailHeadersOf = (raw) => {
|
|
74
|
-
const
|
|
75
|
-
const end = [bytes.indexOf("\r\n\r\n"), bytes.indexOf("\n\n")].filter((at) => at !== -1);
|
|
76
|
-
const head = bytes.subarray(0, end.length ? Math.min(...end) : Math.min(bytes.length, 256 * 1024));
|
|
77
|
-
const unfolded = unfold(head.toString("utf8"));
|
|
81
|
+
const unfolded = unfold(headTextOf(raw));
|
|
78
82
|
const listId = listIdOf(field(unfolded, "List-Id"));
|
|
79
83
|
const precedence = field(unfolded, "Precedence")?.trim().toLowerCase();
|
|
80
84
|
const [replyTo] = addressesOf(field(unfolded, "Reply-To"));
|
package/dist/mail/index.d.ts
CHANGED
|
@@ -12,6 +12,6 @@ export { mapProposedTags, ruleTagger, settleRuleTags, type TagOutcome } from "./
|
|
|
12
12
|
export { conditionKeys, decide, describeMatch, ENVELOPE, envelopeBox, envelopeBoxOf, type MailAction, type MailFacts, type MailRule, type MailRuleMatch, matches, sameMatch, tagsOf, } from "./rules.js";
|
|
13
13
|
export { type DomainAge, FRAUD_SIGNALS, FREE_MAIL, registrableOf, SCORE_RULE, type Scored, type Signal, scoreGroup, WEIGHTS, } from "./score.js";
|
|
14
14
|
export { DEMOTE_AT, type DomainAges, demotedSenders, type LookupResult, SCORE_VERSION, scoreSenders, } from "./sender-scores.js";
|
|
15
|
-
export type { Listed, MailSource, Searched } from "./source.js";
|
|
15
|
+
export type { Listed, MailSource, Quick, Searched } from "./source.js";
|
|
16
16
|
export { listedOf, type StoredMail } from "./stored.js";
|
|
17
17
|
export { MAX_ATTACHMENT_CHARS, type MailAccount, type MailIndexed, type MailSyncOptions, type MailSyncResult, RETRYABLE_ATTACHMENTS, syncMail, } from "./sync.js";
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import type { BudgetStop, RunBudget } from "../background/budget.js";
|
|
2
|
+
import { type Ranges } from "./resume.js";
|
|
3
|
+
import type { Listed, MailSource, Quick, Searched } from "./source.js";
|
|
4
|
+
export declare const FETCH_SLICE = 50;
|
|
5
|
+
export declare const keyOf: (folder: string, uid: string) => string;
|
|
6
|
+
/** What a budgeted run carries from round to round that a quick folder also reads and writes. */
|
|
7
|
+
export interface QuickPlan {
|
|
8
|
+
budget: RunBudget;
|
|
9
|
+
/** UIDs a round already asked for, `folder\0uid`. */
|
|
10
|
+
tried: Set<string>;
|
|
11
|
+
/** Messages the source could not give; asked for once more when the pass is over. */
|
|
12
|
+
failed: Set<string>;
|
|
13
|
+
unlisted: Set<string>;
|
|
14
|
+
/** Only these are asked for: the retry pass. */
|
|
15
|
+
only?: Set<string>;
|
|
16
|
+
/** Each quick folder's listing of what was not done, made once; a round takes the next newest of it. */
|
|
17
|
+
quick: Map<string, Quick[]>;
|
|
18
|
+
stopped: BudgetStop | null;
|
|
19
|
+
remaining: number;
|
|
20
|
+
asked: number;
|
|
21
|
+
}
|
|
22
|
+
export interface QuickRead {
|
|
23
|
+
/** Messages read and identified, with their raw bytes so the save does not read them again. */
|
|
24
|
+
read: {
|
|
25
|
+
item: Listed;
|
|
26
|
+
raw: Uint8Array;
|
|
27
|
+
}[];
|
|
28
|
+
unreadable: {
|
|
29
|
+
message: string;
|
|
30
|
+
reason: string;
|
|
31
|
+
}[];
|
|
32
|
+
/** Mail of the folder is left for a later round or run. */
|
|
33
|
+
capped: boolean;
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* One round of a folder whose server gives no cheap message id: UID, flags, date and size of what is not done, then
|
|
37
|
+
* the raw bytes of the newest of it, whose Message-ID gives the id the full listing would have given.
|
|
38
|
+
*/
|
|
39
|
+
export declare const readQuick: (source: MailSource & Required<Pick<MailSource, "quick" | "identify">>, since: Date, folder: string, searched: Searched, done: Ranges, max: number, plan: QuickPlan | undefined) => Promise<QuickRead>;
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import { headersOfRaw } from "./headers.js";
|
|
2
|
+
import { within } from "./resume.js";
|
|
3
|
+
export const FETCH_SLICE = 50;
|
|
4
|
+
export const keyOf = (folder, uid) => `${folder}\0${uid}`;
|
|
5
|
+
/**
|
|
6
|
+
* One round of a folder whose server gives no cheap message id: UID, flags, date and size of what is not done, then
|
|
7
|
+
* the raw bytes of the newest of it, whose Message-ID gives the id the full listing would have given.
|
|
8
|
+
*/
|
|
9
|
+
export const readQuick = async (source, since, folder, searched, done, max, plan) => {
|
|
10
|
+
const answer = { read: [], unreadable: [], capped: false };
|
|
11
|
+
let items = plan?.quick.get(folder);
|
|
12
|
+
if (items === undefined) {
|
|
13
|
+
const asked = searched.uids.filter((uid) => !within(done, Number(uid)) && (plan?.only === undefined || plan.only.has(keyOf(folder, uid))));
|
|
14
|
+
if (asked.length > 0 && plan) {
|
|
15
|
+
const stop = plan.budget.take();
|
|
16
|
+
if (stop) {
|
|
17
|
+
plan.stopped = stop;
|
|
18
|
+
plan.remaining += asked.length;
|
|
19
|
+
return { ...answer, capped: true };
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
items = asked.length === 0 ? [] : await source.quick(since, folder, { searched: { ...searched, uids: asked } });
|
|
23
|
+
if (plan !== undefined) {
|
|
24
|
+
plan.quick.set(folder, items);
|
|
25
|
+
const got = new Set(items.map(({ uid }) => uid));
|
|
26
|
+
for (const uid of asked)
|
|
27
|
+
if (!got.has(uid)) {
|
|
28
|
+
plan.failed.add(keyOf(folder, uid));
|
|
29
|
+
plan.unlisted.add(keyOf(folder, uid));
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
const open = items
|
|
34
|
+
.filter(({ uid }) => !within(done, Number(uid)) && !plan?.tried.has(keyOf(folder, uid)))
|
|
35
|
+
.sort((a, b) => Number(b.uid) - Number(a.uid));
|
|
36
|
+
const slice = open.slice(0, plan === undefined ? max : (source.fetchBatch ?? FETCH_SLICE));
|
|
37
|
+
if (plan !== undefined)
|
|
38
|
+
plan.remaining += open.length - slice.length;
|
|
39
|
+
answer.capped = open.length > slice.length;
|
|
40
|
+
if (slice.length === 0)
|
|
41
|
+
return answer;
|
|
42
|
+
if (plan !== undefined) {
|
|
43
|
+
const stop = plan.budget.take();
|
|
44
|
+
if (stop) {
|
|
45
|
+
plan.stopped = stop;
|
|
46
|
+
plan.remaining += slice.length;
|
|
47
|
+
return { ...answer, capped: true };
|
|
48
|
+
}
|
|
49
|
+
for (const { uid } of slice)
|
|
50
|
+
plan.tried.add(keyOf(folder, uid));
|
|
51
|
+
plan.asked += slice.length;
|
|
52
|
+
}
|
|
53
|
+
const raws = await source.fetchRaw(slice);
|
|
54
|
+
// Every email failing is the connection or the login, not the mail: stop instead of trying the rest.
|
|
55
|
+
if (slice.length > 1 && raws.every((raw) => raw instanceof Error))
|
|
56
|
+
throw raws[0];
|
|
57
|
+
for (const [index, item] of slice.entries()) {
|
|
58
|
+
const raw = raws[index];
|
|
59
|
+
const key = keyOf(folder, item.uid);
|
|
60
|
+
if (raw === undefined || raw instanceof Error) {
|
|
61
|
+
if (plan === undefined)
|
|
62
|
+
answer.unreadable.push({ message: `uid:${folder}:${item.uid}`, reason: raw?.message ?? "not read" });
|
|
63
|
+
else {
|
|
64
|
+
plan.failed.add(key);
|
|
65
|
+
plan.unlisted.add(key);
|
|
66
|
+
}
|
|
67
|
+
continue;
|
|
68
|
+
}
|
|
69
|
+
plan?.unlisted.delete(key);
|
|
70
|
+
answer.read.push({ item: { ...item, ...source.identify(raw, item), headers: headersOfRaw(raw) }, raw });
|
|
71
|
+
}
|
|
72
|
+
return answer;
|
|
73
|
+
};
|
package/dist/mail/source.d.ts
CHANGED
|
@@ -16,10 +16,17 @@ export interface Listed {
|
|
|
16
16
|
/** Read only when asked for; absent when not asked or unreadable, so no rule applies to it. */
|
|
17
17
|
headers?: MailHeaders;
|
|
18
18
|
}
|
|
19
|
+
/** A message as `quick` lists it: everything but the ids, which a source without a cheap one reads from the raw bytes. */
|
|
20
|
+
export type Quick = Omit<Listed, "messageId" | "threadId">;
|
|
19
21
|
export interface Searched {
|
|
20
22
|
/** Absent when the server does not say; then no UID is trusted across runs. */
|
|
21
23
|
validity?: string;
|
|
22
24
|
uids: string[];
|
|
25
|
+
/**
|
|
26
|
+
* The server gives no message id without reading the message, so `list` is slow. The sync then lists with `quick`
|
|
27
|
+
* and takes the id from the raw bytes with `identify`.
|
|
28
|
+
*/
|
|
29
|
+
quick?: boolean;
|
|
23
30
|
}
|
|
24
31
|
/** Everything the import asks of a mail client, so a transport can be replaced without touching the sync. */
|
|
25
32
|
export interface MailSource {
|
|
@@ -43,6 +50,18 @@ export interface MailSource {
|
|
|
43
50
|
* those. Absent for a file, which has no UIDs.
|
|
44
51
|
*/
|
|
45
52
|
search?(since: Date, folder: string): Promise<Searched>;
|
|
53
|
+
/**
|
|
54
|
+
* For a folder whose `search` said `quick`: UID, flags, date and size of the given UIDs, with no envelope and no
|
|
55
|
+
* header. A UID the server does not return is left out.
|
|
56
|
+
*/
|
|
57
|
+
quick?(since: Date, folder: string, options: {
|
|
58
|
+
searched: Searched;
|
|
59
|
+
}): Promise<Quick[]>;
|
|
60
|
+
/** The ids of a message read by `fetchRaw`, worked out as `list` works them out; needed wherever `quick` is. */
|
|
61
|
+
identify?(raw: Uint8Array, item: Quick): {
|
|
62
|
+
messageId: string;
|
|
63
|
+
threadId: string;
|
|
64
|
+
};
|
|
46
65
|
/** A file's size and modification time, so an unchanged file read to the end is not read again. */
|
|
47
66
|
stamp?(): Promise<FileStamp>;
|
|
48
67
|
/**
|
|
@@ -70,5 +89,5 @@ export interface MailSource {
|
|
|
70
89
|
/** Puts labels back on messages in `folder`, by Gmail id. */
|
|
71
90
|
label?(messageIds: readonly string[], folder: string, labels: readonly string[]): Promise<void>;
|
|
72
91
|
/** Each item's raw RFC 5322 bytes, in order; one that could not be read is an Error, not a failed call. */
|
|
73
|
-
fetchRaw(items:
|
|
92
|
+
fetchRaw(items: Quick[]): Promise<(Uint8Array | Error)[]>;
|
|
74
93
|
}
|
package/dist/mail/sync.js
CHANGED
|
@@ -6,6 +6,7 @@ import { GMAIL_ALL_MAIL, mailFolders } from "./folders.js";
|
|
|
6
6
|
import { dropMissing, tooManyGone } from "./gone.js";
|
|
7
7
|
import { mailHeadersOf } from "./headers.js";
|
|
8
8
|
import { MAX_BODY_CHARS, parseMail } from "./parse.js";
|
|
9
|
+
import { FETCH_SLICE, keyOf, readQuick } from "./quick.js";
|
|
9
10
|
import { FULL_PASS_MS, MBOX_RESUME, mergeRanges, parsePoint, rangesOf, resumeKey, rulesKey, within, } from "./resume.js";
|
|
10
11
|
import { ruleTagger, settleRuleTags } from "./rule-tags.js";
|
|
11
12
|
import { decide, ENVELOPE, envelopeBox, envelopeBoxOf, tagsOf } from "./rules.js";
|
|
@@ -54,8 +55,6 @@ const readAttachments = async (mail, messageId, { loadEngine, maxAttachmentChars
|
|
|
54
55
|
return files;
|
|
55
56
|
};
|
|
56
57
|
const LIST_SLICE = 200;
|
|
57
|
-
const FETCH_SLICE = 50;
|
|
58
|
-
const keyOf = (folder, uid) => `${folder}\0${uid}`;
|
|
59
58
|
export const syncMail = async (source, store, options) => {
|
|
60
59
|
const { budget, index } = options;
|
|
61
60
|
if (budget === undefined || options.retryAttachments || options.account.mode === "mbox")
|
|
@@ -68,6 +67,8 @@ export const syncMail = async (source, store, options) => {
|
|
|
68
67
|
modes: new Map(),
|
|
69
68
|
unreadable: new Map(),
|
|
70
69
|
unlisted: new Set(),
|
|
70
|
+
quick: new Map(),
|
|
71
|
+
quickFolders: new Set(),
|
|
71
72
|
stopped: null,
|
|
72
73
|
remaining: 0,
|
|
73
74
|
asked: 0,
|
|
@@ -100,6 +101,7 @@ export const syncMail = async (source, store, options) => {
|
|
|
100
101
|
// A message the server failed on once is often fine a minute later; only one more try, then the next run.
|
|
101
102
|
if (plan.stopped === null && plan.failed.size > 0) {
|
|
102
103
|
plan.only = new Set(plan.failed);
|
|
104
|
+
plan.quick.clear();
|
|
103
105
|
plan.failed.clear();
|
|
104
106
|
plan.tried.clear();
|
|
105
107
|
await goOn();
|
|
@@ -188,6 +190,17 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
|
|
|
188
190
|
mboxPoint = point;
|
|
189
191
|
}
|
|
190
192
|
let capped = false;
|
|
193
|
+
const unreadable = [];
|
|
194
|
+
/** Raw mail read to learn its id, kept for the save so it is not read twice. */
|
|
195
|
+
const rawOf = new Map();
|
|
196
|
+
const quickRan = new Set();
|
|
197
|
+
let holding;
|
|
198
|
+
const holdsMail = async (folder) => {
|
|
199
|
+
holding ??= new Set((await store.mail.emailRefs({ accountId })).flatMap(({ mailboxes }) => mailboxes
|
|
200
|
+
.filter(({ kind }) => kind !== "label" && kind !== ENVELOPE && kind !== BULK)
|
|
201
|
+
.map((box) => box.externalId)));
|
|
202
|
+
return holding.has(folder);
|
|
203
|
+
};
|
|
191
204
|
for (const folder of folders) {
|
|
192
205
|
if (mboxPoint !== undefined)
|
|
193
206
|
break;
|
|
@@ -212,6 +225,38 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
|
|
|
212
225
|
if (kept !== undefined)
|
|
213
226
|
storedPoints.set(folder, kept);
|
|
214
227
|
const point = fresh ? undefined : kept;
|
|
228
|
+
const sameRules = point !== undefined && point.validity === searched.validity && point.rules === ruleSet;
|
|
229
|
+
// Without a cheap id the mail's identity is in its bytes, so a folder is listed quickly only when no stored mail
|
|
230
|
+
// can be read twice: its done UIDs are known, or the store holds nothing from it. Otherwise (a new UIDVALIDITY,
|
|
231
|
+
// other rules, --rescan) the full header listing recognises what is stored without downloading it.
|
|
232
|
+
const quickable = !snapshot &&
|
|
233
|
+
searched.quick === true &&
|
|
234
|
+
searched.validity !== undefined &&
|
|
235
|
+
source.quick !== undefined &&
|
|
236
|
+
source.identify !== undefined;
|
|
237
|
+
const inQuick = quickable &&
|
|
238
|
+
(plan === undefined
|
|
239
|
+
? sameRules || !(await holdsMail(folder))
|
|
240
|
+
: plan.quickFolders.has(folder) || (!plan.modes.has(folder) && (sameRules || !(await holdsMail(folder)))));
|
|
241
|
+
if (inQuick) {
|
|
242
|
+
plan?.quickFolders.add(folder);
|
|
243
|
+
plan?.modes.set(folder, "resumed");
|
|
244
|
+
resumed.add(folder);
|
|
245
|
+
quickRan.add(folder);
|
|
246
|
+
const done = point !== undefined && sameRules ? point.done : [];
|
|
247
|
+
if (point !== undefined && sameRules)
|
|
248
|
+
earlier.set(folder, point);
|
|
249
|
+
const round = await readQuick(source, since, folder, searched, done, max, plan);
|
|
250
|
+
if (round.capped)
|
|
251
|
+
capped = true;
|
|
252
|
+
for (const { item, raw } of round.read) {
|
|
253
|
+
collected.push(item);
|
|
254
|
+
rawOf.set(item, raw);
|
|
255
|
+
}
|
|
256
|
+
for (const bad of round.unreadable)
|
|
257
|
+
unreadable.push(bad);
|
|
258
|
+
continue;
|
|
259
|
+
}
|
|
215
260
|
if (point !== undefined && point.validity === searched.validity && point.rules === ruleSet) {
|
|
216
261
|
earlier.set(folder, point);
|
|
217
262
|
const mode = plan?.modes.get(folder);
|
|
@@ -301,6 +346,8 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
|
|
|
301
346
|
skipped = "an mbox file is a snapshot; mail missing from it is not proven deleted";
|
|
302
347
|
else if (plan !== undefined && (capped || plan.tried.size > plan.asked || plan.only !== undefined))
|
|
303
348
|
skipped = "this run read the window in parts; a run that lists all of it at once checks deletions";
|
|
349
|
+
else if (quickRan.size > 0)
|
|
350
|
+
skipped = "this run listed only mail not read before, by UID; --rescan lists every message and checks deletions";
|
|
304
351
|
else if (resumed.size > 0)
|
|
305
352
|
skipped =
|
|
306
353
|
"this run listed only mail not read before; a full listing, at least daily or with --rescan, checks deletions";
|
|
@@ -339,7 +386,6 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
|
|
|
339
386
|
await store.mail.setMailboxes(accountId, [...scanned.values()], new Map([...labelsOf].map(([id, { labels }]) => [id, (labels ?? []).map((name) => label(name).externalId)])), startedAt);
|
|
340
387
|
}
|
|
341
388
|
const attachments = [];
|
|
342
|
-
const unreadable = [];
|
|
343
389
|
let saved = 0;
|
|
344
390
|
let alreadyStored = 0;
|
|
345
391
|
let ruledOut = 0;
|
|
@@ -509,7 +555,9 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
|
|
|
509
555
|
for (let start = 0; start < pending.length; start += fetchChunk) {
|
|
510
556
|
await stopIfAborted();
|
|
511
557
|
const chunk = pending.slice(start, start + fetchChunk);
|
|
512
|
-
|
|
558
|
+
// Mail read to learn its id is already in hand: a chunk of it costs no request.
|
|
559
|
+
const unread = chunk.filter((item) => !rawOf.has(item));
|
|
560
|
+
if (plan !== undefined && unread.length > 0) {
|
|
513
561
|
const stop = plan.budget.take();
|
|
514
562
|
if (stop) {
|
|
515
563
|
plan.stopped = stop;
|
|
@@ -519,7 +567,8 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
|
|
|
519
567
|
}
|
|
520
568
|
let raws;
|
|
521
569
|
try {
|
|
522
|
-
|
|
570
|
+
const got = unread.length > 0 ? await source.fetchRaw(unread) : [];
|
|
571
|
+
raws = chunk.map((item) => rawOf.get(item) ?? got[unread.indexOf(item)]);
|
|
523
572
|
}
|
|
524
573
|
catch (error) {
|
|
525
574
|
// What this run saved stays saved: the next one goes on from it.
|
|
@@ -601,7 +650,7 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
|
|
|
601
650
|
await store.mail.setFlags(accountId, new Map([...labelsOf].flatMap(([id, { seen, flagged }]) => seen === undefined && flagged === undefined
|
|
602
651
|
? []
|
|
603
652
|
: [[id, { ...(seen === undefined ? {} : { read: seen }), ...(flagged === undefined ? {} : { flagged }) }]])), startedAt);
|
|
604
|
-
const incomplete = plan !== undefined && (
|
|
653
|
+
const incomplete = capped || (plan !== undefined && (plan.stopped !== null || plan.failed.size > 0));
|
|
605
654
|
const complete = !full && !incomplete && unreadable.length === 0 && (plan?.unreadable.size ?? 0) === 0;
|
|
606
655
|
const bulk = new Map(listed.filter((item) => item.headers?.bulk).map((item) => [item.messageId, [bulkBox.externalId]]));
|
|
607
656
|
if (bulk.size)
|
|
@@ -53,6 +53,28 @@ export const mailSourceContract = (make) => {
|
|
|
53
53
|
});
|
|
54
54
|
assert.deepEqual(listed.map(({ uid }) => uid), first === undefined ? [] : [first]);
|
|
55
55
|
}),
|
|
56
|
+
test("lists quickly the same UIDs, flags and sizes, and gives from the bytes the ids `list` gives", async (source) => {
|
|
57
|
+
if (source.search === undefined || source.quick === undefined || source.identify === undefined)
|
|
58
|
+
return;
|
|
59
|
+
const searched = await source.search(SINCE, folder);
|
|
60
|
+
if (!searched.quick)
|
|
61
|
+
return;
|
|
62
|
+
const quick = await source.quick(SINCE, folder, { searched });
|
|
63
|
+
const full = await source.list(SINCE, folder);
|
|
64
|
+
assert.deepEqual(quick.map(({ uid }) => uid).sort(), full.map(({ uid }) => uid).sort());
|
|
65
|
+
const raws = await source.fetchRaw(quick);
|
|
66
|
+
for (const [index, item] of quick.entries()) {
|
|
67
|
+
const raw = raws[index];
|
|
68
|
+
assert.ok(raw instanceof Uint8Array, "bytes");
|
|
69
|
+
const same = full.find(({ uid }) => uid === item.uid);
|
|
70
|
+
assert.deepEqual(source.identify(raw, item), {
|
|
71
|
+
messageId: same?.messageId,
|
|
72
|
+
threadId: same?.threadId,
|
|
73
|
+
});
|
|
74
|
+
assert.equal(item.seen, same?.seen);
|
|
75
|
+
assert.equal(item.receivedAt, same?.receivedAt);
|
|
76
|
+
}
|
|
77
|
+
}),
|
|
56
78
|
test("returns each message's bytes in order, and one that cannot be read as an Error, not a failed call", async (source) => {
|
|
57
79
|
const listed = await source.list(SINCE, folder);
|
|
58
80
|
const raws = await source.fetchRaw(listed);
|
|
@@ -48,7 +48,12 @@ export interface FakeMailbox {
|
|
|
48
48
|
search: number;
|
|
49
49
|
list: number;
|
|
50
50
|
fetchRaw: number;
|
|
51
|
+
quick?: number;
|
|
51
52
|
};
|
|
53
|
+
/** Answers as a server with no cheap message id does: `search` says `quick`, and `quick` and `identify` work. */
|
|
54
|
+
quick?: boolean;
|
|
55
|
+
/** The UIDs `fetchRaw` was asked for, in order, so a test proves nothing is read twice. */
|
|
56
|
+
read?: string[];
|
|
52
57
|
/** What the source tells a budgeted sync about its batch sizes. */
|
|
53
58
|
listBatch?: number;
|
|
54
59
|
fetchBatch?: number;
|
|
@@ -29,18 +29,21 @@ export const rawOf = (m) => {
|
|
|
29
29
|
};
|
|
30
30
|
const failure = (message) => new CliError("provider_error", message, { reason: "mail.source-failed" });
|
|
31
31
|
const labelsOf = (m) => [...new Set([...(m.labels ?? []), ...(m.flagged ? ["Starred"] : [])])].sort();
|
|
32
|
-
const
|
|
32
|
+
const quickOf = (m, folder) => ({
|
|
33
33
|
uid: m.uid,
|
|
34
|
-
messageId: m.messageId,
|
|
35
|
-
threadId: m.threadId,
|
|
36
34
|
receivedAt: new Date(m.receivedAt).toISOString(),
|
|
37
35
|
size: rawOf(m).length,
|
|
38
36
|
labels: labelsOf(m),
|
|
39
37
|
seen: m.seen ?? false,
|
|
40
38
|
flagged: m.flagged ?? false,
|
|
41
|
-
...(headers ? { headers: headersOf(Buffer.from(rawOf(m)).toString().split("\r\n\r\n")[0] ?? "") } : {}),
|
|
42
39
|
folder,
|
|
43
40
|
});
|
|
41
|
+
const listingOf = (m, headers, folder) => ({
|
|
42
|
+
...quickOf(m, folder),
|
|
43
|
+
messageId: m.messageId,
|
|
44
|
+
threadId: m.threadId,
|
|
45
|
+
...(headers ? { headers: headersOf(Buffer.from(rawOf(m)).toString().split("\r\n\r\n")[0] ?? "") } : {}),
|
|
46
|
+
});
|
|
44
47
|
/**
|
|
45
48
|
* An in-memory mail source with Gmail's shape: ids and threads that outlive a folder, labels and flags, UIDVALIDITY,
|
|
46
49
|
* a trash a move reaches, and All Mail under the account's language. The sync tests and the adapter contract run on it.
|
|
@@ -59,6 +62,7 @@ export const fakeMailSource = (mailbox) => {
|
|
|
59
62
|
const from = Date.UTC(since.getUTCFullYear(), since.getUTCMonth(), since.getUTCDate());
|
|
60
63
|
return {
|
|
61
64
|
validity: String(mailbox.validity ?? 12),
|
|
65
|
+
...(mailbox.quick ? { quick: true } : {}),
|
|
62
66
|
uids: inFolder(folder)
|
|
63
67
|
.filter((m) => Date.parse(m.receivedAt) >= from)
|
|
64
68
|
.map((m) => m.uid),
|
|
@@ -73,7 +77,7 @@ export const fakeMailSource = (mailbox) => {
|
|
|
73
77
|
};
|
|
74
78
|
const count = (kind) => {
|
|
75
79
|
if (mailbox.calls)
|
|
76
|
-
mailbox.calls[kind]
|
|
80
|
+
mailbox.calls[kind] = (mailbox.calls[kind] ?? 0) + 1;
|
|
77
81
|
};
|
|
78
82
|
return {
|
|
79
83
|
...(mailbox.listBatch === undefined ? {} : { listBatch: mailbox.listBatch }),
|
|
@@ -94,6 +98,24 @@ export const fakeMailSource = (mailbox) => {
|
|
|
94
98
|
.filter((m) => wanted.has(m.uid) && !failsNow(m.uid))
|
|
95
99
|
.map((m) => listingOf(m, options?.headers ?? false, folder));
|
|
96
100
|
},
|
|
101
|
+
...(mailbox.quick
|
|
102
|
+
? {
|
|
103
|
+
quick: async (_since, folder, options) => {
|
|
104
|
+
count("quick");
|
|
105
|
+
if (mailbox.searchFails)
|
|
106
|
+
throw failure(mailbox.searchFails);
|
|
107
|
+
const wanted = new Set(options.searched.uids);
|
|
108
|
+
return inFolder(folder)
|
|
109
|
+
.filter((m) => wanted.has(m.uid) && !failsNow(m.uid))
|
|
110
|
+
.map((m) => quickOf(m, folder));
|
|
111
|
+
},
|
|
112
|
+
identify: (raw) => {
|
|
113
|
+
const id = /^Message-ID:\s*<(.+?)@example\.test>/im.exec(Buffer.from(raw).toString())?.[1];
|
|
114
|
+
const m = mailbox.messages.find((each) => each.messageId === id);
|
|
115
|
+
return { messageId: m.messageId, threadId: m.threadId };
|
|
116
|
+
},
|
|
117
|
+
}
|
|
118
|
+
: {}),
|
|
97
119
|
find: async (messageIds, folder, options) => {
|
|
98
120
|
for (const id of messageIds)
|
|
99
121
|
mailbox.asked?.push(`msgid:${id}`);
|
|
@@ -131,6 +153,7 @@ export const fakeMailSource = (mailbox) => {
|
|
|
131
153
|
},
|
|
132
154
|
fetchRaw: async (items) => {
|
|
133
155
|
count("fetchRaw");
|
|
156
|
+
mailbox.read?.push(...items.map(({ uid }) => uid));
|
|
134
157
|
return items.map((item) => {
|
|
135
158
|
const m = mailbox.messages.find((each) => each.uid === item.uid);
|
|
136
159
|
if (m === undefined || mailbox.unreadable?.includes(item.uid) || failsNow(item.uid))
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@wirecat/cli-core",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.487.0",
|
|
4
4
|
"description": "The messenger-neutral half of a messaging CLI: domain model, message locators, rendering, name resolution, the SQLite seam, the command skeleton and shared read commands, the send guard, run records and the message store",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Viacheslav Ptsarev",
|