@wirecat/cli-core 0.540.0 → 0.542.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +32 -0
- package/dist/background/schedules.d.ts +3 -1
- package/dist/background/schedules.js +18 -3
- package/dist/cli/messenger/search-backend-options.d.ts +1 -1
- package/dist/cli/messenger/search-backend-options.js +2 -2
- package/dist/cli/schedule/schedule-command.js +17 -3
- package/dist/errors/reasons.js +1 -0
- package/dist/mail/attachments.d.ts +2 -1
- package/dist/mail/attachments.js +4 -2
- package/dist/mail/cli/index.d.ts +2 -0
- package/dist/mail/cli/index.js +11 -4
- package/dist/mail/server-search.d.ts +5 -0
- package/dist/mail/server-search.js +70 -52
- package/dist/mail/sync.js +14 -9
- package/dist/oauth/authorize.js +21 -5
- package/dist/oauth/provider.d.ts +3 -1
- package/dist/services/server-search.d.ts +4 -0
- package/dist/services/server-search.js +2 -1
- package/dist/store/sqlite/emails.d.ts +2 -0
- package/dist/store/sqlite/emails.js +2 -0
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -7,6 +7,38 @@ shape of this file.
|
|
|
7
7
|
|
|
8
8
|
## Unreleased
|
|
9
9
|
|
|
10
|
+
## 0.542.0 — 02.11.2026
|
|
11
|
+
|
|
12
|
+
### Added
|
|
13
|
+
|
|
14
|
+
- **`redirect: { kind: "loopback", host: "localhost" }` signs in through `http://localhost:<port>/callback`.** Microsoft
|
|
15
|
+
Entra ignores the port only for `localhost` redirect URIs, so a public-client app registered as
|
|
16
|
+
`http://localhost/callback` needs it. The listener then takes `127.0.0.1` and, when the host has IPv6, `::1`, because
|
|
17
|
+
the browser may resolve `localhost` to either. Without `host` nothing changes: Google and the rest keep `127.0.0.1`.
|
|
18
|
+
|
|
19
|
+
## 0.541.0 — 02.11.2026
|
|
20
|
+
|
|
21
|
+
### Added
|
|
22
|
+
|
|
23
|
+
- **`MailServerAccess.timeMs`**: a mailbox's own default for the server search's time bound, for a server whose
|
|
24
|
+
search is slow; `--server-time` still wins. `MailReads.serverTimeDefault` puts it in the `--server-time` help.
|
|
25
|
+
- **`EmailFilter.externalIds`**: only these Message-IDs, for `emails`, `emailRefs` and `emailFacts`.
|
|
26
|
+
|
|
27
|
+
### Fixed
|
|
28
|
+
|
|
29
|
+
- **`schedule exec` and `schedule run` wait for their child as long as the child's own deadline allows**, instead of
|
|
30
|
+
answering `timeout` (exit 9) at 30 s while the child went on: a scheduled sync of more than 30 s failed every time
|
|
31
|
+
and kept writing after the answer. A signal or an explicit `--timeout` now stops the child (SIGTERM, then SIGKILL
|
|
32
|
+
after 10 s), and its exit code is the run's.
|
|
33
|
+
- **A mail sync no longer loads every stored email of the account in each round**: it asks the store about the mail
|
|
34
|
+
the round listed. On 260,000 stored emails and a 4,900-message window (`bench/mail-sync-scale`), from 17.7 s and
|
|
35
|
+
1.1 GB peak memory to 0.3 s and 190 MB.
|
|
36
|
+
- **A mail sync that is stopped while a listing is in flight writes nothing for it**, and reading attachments stops
|
|
37
|
+
at the stop.
|
|
38
|
+
- **The mail server search keeps parsing and attachment reading inside its time bound**, ends that bound before the
|
|
39
|
+
command's deadline so it answers in part instead of timing out, and stops writing once the command is stopped. A
|
|
40
|
+
find larger than 5 MB is left on the server unread, and the answer says so (`search.server-too-large`).
|
|
41
|
+
|
|
10
42
|
## 0.540.0 — 02.11.2026
|
|
11
43
|
|
|
12
44
|
### Added
|
|
@@ -102,7 +102,9 @@ export declare const findProgram: (name: string, env: NodeJS.ProcessEnv, platfor
|
|
|
102
102
|
* the last result in `last.json`. An agent runs under `<PREFIX>_READ_ONLY`, so nothing it is talked
|
|
103
103
|
* into can send. A one-off removes itself afterwards.
|
|
104
104
|
*/
|
|
105
|
-
export declare const runSchedule: (app: AppIdentity, system: ServerSystem, env: NodeJS.ProcessEnv, file: string, { now, timer }?: {
|
|
105
|
+
export declare const runSchedule: (app: AppIdentity, system: ServerSystem, env: NodeJS.ProcessEnv, file: string, { now, timer, signal, killAfterMs, }?: {
|
|
106
106
|
now?: () => Date;
|
|
107
107
|
timer?: Timer;
|
|
108
|
+
signal?: AbortSignal;
|
|
109
|
+
killAfterMs?: number;
|
|
108
110
|
}) => Promise<LastRun>;
|
|
@@ -326,12 +326,14 @@ export const findProgram = (name, env, platform) => {
|
|
|
326
326
|
return undefined;
|
|
327
327
|
};
|
|
328
328
|
const KEEP_RUNS = 30;
|
|
329
|
+
/** How long a stopped child has to save and answer after SIGTERM before it is killed. */
|
|
330
|
+
const KILL_AFTER_MS = 10_000;
|
|
329
331
|
/**
|
|
330
332
|
* One run: the command or the agent with the schedule's variables, its output kept beside the schedule,
|
|
331
333
|
* the last result in `last.json`. An agent runs under `<PREFIX>_READ_ONLY`, so nothing it is talked
|
|
332
334
|
* into can send. A one-off removes itself afterwards.
|
|
333
335
|
*/
|
|
334
|
-
export const runSchedule = async (app, system, env, file, { now = () => new Date(), timer } = {}) => {
|
|
336
|
+
export const runSchedule = async (app, system, env, file, { now = () => new Date(), timer, signal, killAfterMs = KILL_AFTER_MS, } = {}) => {
|
|
335
337
|
const schedule = readSchedule(file);
|
|
336
338
|
const folder = join(file, "..");
|
|
337
339
|
const runs = join(folder, "runs");
|
|
@@ -369,11 +371,24 @@ export const runSchedule = async (app, system, env, file, { now = () => new Date
|
|
|
369
371
|
err.push(Buffer.from(String(error)));
|
|
370
372
|
resolve(127);
|
|
371
373
|
});
|
|
372
|
-
child
|
|
374
|
+
// The child keeps its own deadline; a stop of this run stops the child too, so nothing it does outlives the run.
|
|
375
|
+
let kill;
|
|
376
|
+
const stop = () => {
|
|
377
|
+
child.kill("SIGTERM");
|
|
378
|
+
kill = setTimeout(() => child.kill("SIGKILL"), killAfterMs);
|
|
379
|
+
};
|
|
380
|
+
if (signal?.aborted)
|
|
381
|
+
stop();
|
|
382
|
+
else
|
|
383
|
+
signal?.addEventListener("abort", stop, { once: true });
|
|
384
|
+
child.on("close", (code, killedBy) => {
|
|
385
|
+
signal?.removeEventListener("abort", stop);
|
|
386
|
+
if (kill)
|
|
387
|
+
clearTimeout(kill);
|
|
373
388
|
writeFileSync(output, Buffer.concat(agent ? out : [...out, ...err]), { mode: 0o600 });
|
|
374
389
|
if (agent)
|
|
375
390
|
writeFileSync(errors, Buffer.concat(err), { mode: 0o600 });
|
|
376
|
-
resolve(code ?? 1);
|
|
391
|
+
resolve(code ?? (killedBy === "SIGKILL" ? 137 : killedBy === "SIGTERM" ? 143 : 1));
|
|
377
392
|
});
|
|
378
393
|
});
|
|
379
394
|
const ended = now();
|
|
@@ -2,7 +2,7 @@ import type { Command } from "commander";
|
|
|
2
2
|
import type { Backend, ServerOptions, ServerSearched } from "../../services/server-search.js";
|
|
3
3
|
import type { Messenger, MessengerContext } from "./context.js";
|
|
4
4
|
/** `--backend` and `--server-time`, for any search whose source has a server that searches: `where` names it. */
|
|
5
|
-
export declare const serverSearchOptions: (command: Command, where: string) => Command;
|
|
5
|
+
export declare const serverSearchOptions: (command: Command, where: string, defaultTime?: string) => Command;
|
|
6
6
|
/** Mounted only where the messenger's server can search messages. */
|
|
7
7
|
export declare const backendOptions: (command: Command, messenger: Messenger) => Command;
|
|
8
8
|
export declare const backendRequest: (command: Command) => {
|
|
@@ -6,9 +6,9 @@ const backendOf = (value) => {
|
|
|
6
6
|
return value;
|
|
7
7
|
};
|
|
8
8
|
/** `--backend` and `--server-time`, for any search whose source has a server that searches: `where` names it. */
|
|
9
|
-
export const serverSearchOptions = (command, where) => command
|
|
9
|
+
export const serverSearchOptions = (command, where, defaultTime = "5s") => command
|
|
10
10
|
.option("--backend <archive|server|both>", `where to search: the local archive, ${where}, or both`, backendOf)
|
|
11
|
-
.option("--server-time <duration>",
|
|
11
|
+
.option("--server-time <duration>", `stop waiting for the server after this long (default: ${defaultTime})`);
|
|
12
12
|
/** Mounted only where the messenger's server can search messages. */
|
|
13
13
|
export const backendOptions = (command, messenger) => messenger.serverSearch
|
|
14
14
|
? serverSearchOptions(command, "the messenger's server (default: both; message discovery uses archive only)")
|
|
@@ -45,6 +45,19 @@ const whenOf = ({ every, at, atTime }, minimum) => {
|
|
|
45
45
|
return { every: minutes };
|
|
46
46
|
};
|
|
47
47
|
/** A schedule as answered: snake_case, but its environment keeps the variables' own names. */
|
|
48
|
+
/**
|
|
49
|
+
* A run waits for its child, which keeps its own deadline (a sync's is 15 minutes): the wait has none of its own, or the
|
|
50
|
+
* run answered `timeout` at 30 s while the child went on. A signal or an explicit --timeout stops the child.
|
|
51
|
+
*/
|
|
52
|
+
const untilChildEnds = async (command, wait) => {
|
|
53
|
+
const environment = environmentOf(command);
|
|
54
|
+
try {
|
|
55
|
+
return await wait(environment.untilStopped?.());
|
|
56
|
+
}
|
|
57
|
+
finally {
|
|
58
|
+
environment.endUntilStopped?.();
|
|
59
|
+
}
|
|
60
|
+
};
|
|
48
61
|
const scheduleAnswer = ({ environment, createdAt, ...schedule }, extra) => ({
|
|
49
62
|
...snakeKeys(schedule),
|
|
50
63
|
environment,
|
|
@@ -222,9 +235,10 @@ export const scheduleCommandFor = ({ app, context: contextOf }) => {
|
|
|
222
235
|
.argument("<name>", "the schedule's name, as `schedule list` shows it")
|
|
223
236
|
.action(async function (name) {
|
|
224
237
|
const { context, system, state, say, timer } = prepare(this);
|
|
225
|
-
const last = await runSchedule(app, system, context.env, scheduleFile(state, scheduleName(name)), {
|
|
238
|
+
const last = await untilChildEnds(this, (signal) => runSchedule(app, system, context.env, scheduleFile(state, scheduleName(name)), {
|
|
226
239
|
timer: timer(),
|
|
227
|
-
|
|
240
|
+
...(signal ? { signal } : {}),
|
|
241
|
+
}));
|
|
228
242
|
say(snakeKeys({ name, ...last }), [`${name}: exit ${last.exitCode} — ${last.output}`]);
|
|
229
243
|
if (last.exitCode !== 0)
|
|
230
244
|
setExitCode(this, 1);
|
|
@@ -253,7 +267,7 @@ export const scheduleCommandFor = ({ app, context: contextOf }) => {
|
|
|
253
267
|
.action(async function (file) {
|
|
254
268
|
// A timer starts it with no profile named; the schedule's own environment carries the profile.
|
|
255
269
|
const { context, system, timer } = prepare(this, { needsProfile: false });
|
|
256
|
-
const last = await runSchedule(app, system, context.env, file, { timer: timer() });
|
|
270
|
+
const last = await untilChildEnds(this, (signal) => runSchedule(app, system, context.env, file, { timer: timer(), ...(signal ? { signal } : {}) }));
|
|
257
271
|
if (last.exitCode !== 0)
|
|
258
272
|
setExitCode(this, 1);
|
|
259
273
|
});
|
package/dist/errors/reasons.js
CHANGED
|
@@ -89,6 +89,7 @@ const OTHERS = [
|
|
|
89
89
|
["search.server-failed", "provider_unavailable", "the server did not answer a search"],
|
|
90
90
|
["search.server-timeout", "timeout", "the server did not answer within --server-time"],
|
|
91
91
|
["search.server-capped", "outcome_unknown", "the server found more than one search reads, or a branch was not sent"],
|
|
92
|
+
["search.server-too-large", "outcome_unknown", "a message the server found is too large to download during a search"],
|
|
92
93
|
validation("store.refused", "the local store refused the document text"),
|
|
93
94
|
["store.not-writable", "permission_error", "the message store cannot be written from this process"],
|
|
94
95
|
[
|
|
@@ -8,8 +8,9 @@ export declare const FETCH_CHUNK = 50;
|
|
|
8
8
|
export declare const addressOf: ({ email, name }: Address) => EmailAddress;
|
|
9
9
|
export type AttachmentLog = NonNullable<MailSyncResult["attachments"]>;
|
|
10
10
|
/** A parsed email's attachments as the store keeps them, each with its text or why it has none. */
|
|
11
|
-
export declare const readAttachments: (mail: ParsedMail, messageId: string, { loadEngine, maxAttachmentChars, at }: {
|
|
11
|
+
export declare const readAttachments: (mail: ParsedMail, messageId: string, { loadEngine, maxAttachmentChars, at, signal, }: {
|
|
12
12
|
loadEngine: LoadEngine;
|
|
13
13
|
maxAttachmentChars: number;
|
|
14
14
|
at: number;
|
|
15
|
+
signal?: AbortSignal;
|
|
15
16
|
}, log: AttachmentLog) => Promise<AttachmentInput[]>;
|
package/dist/mail/attachments.js
CHANGED
|
@@ -4,12 +4,14 @@ export const MAX_ATTACHMENT_CHARS = 200_000;
|
|
|
4
4
|
export const FETCH_CHUNK = 50;
|
|
5
5
|
export const addressOf = ({ email, name }) => ({ address: email, name });
|
|
6
6
|
/** A parsed email's attachments as the store keeps them, each with its text or why it has none. */
|
|
7
|
-
export const readAttachments = async (mail, messageId, { loadEngine, maxAttachmentChars, at }, log) => {
|
|
7
|
+
export const readAttachments = async (mail, messageId, { loadEngine, maxAttachmentChars, at, signal, }, log) => {
|
|
8
8
|
const files = [];
|
|
9
9
|
for (const [position, part] of mail.attachments.entries()) {
|
|
10
|
+
if (signal?.aborted)
|
|
11
|
+
throw signal.reason;
|
|
10
12
|
const extraction = part.bytes === null
|
|
11
13
|
? { status: "unavailable" }
|
|
12
|
-
: await extractText(part.bytes, { kind: "file", name: part.name, mime: part.mime, path: part.name }, loadEngine);
|
|
14
|
+
: await extractText(part.bytes, { kind: "file", name: part.name, mime: part.mime, path: part.name }, loadEngine, signal);
|
|
13
15
|
log.push({ message: messageId, part: part.id, status: extraction.status });
|
|
14
16
|
const extracted = extraction.status === "extracted" ? extraction : undefined;
|
|
15
17
|
files.push({
|
package/dist/mail/cli/index.d.ts
CHANGED
|
@@ -12,6 +12,8 @@ export interface MailReads {
|
|
|
12
12
|
read: <T>(command: Command, work: (store: MessageStore, mailbox: AccountKey) => Promise<T>) => Promise<T>;
|
|
13
13
|
/** The mailbox's server, for `search messages --backend`; without it the search reads the store only. */
|
|
14
14
|
server?: (command: Command) => MailServerAccess;
|
|
15
|
+
/** How `--server-time` help states its default when a provider's own differs (`MailServerAccess.timeMs`). */
|
|
16
|
+
serverTimeDefault?: string;
|
|
15
17
|
}
|
|
16
18
|
export declare const THREAD_QUERY: {
|
|
17
19
|
readonly entity: "mail_thread";
|
package/dist/mail/cli/index.js
CHANGED
|
@@ -4,7 +4,7 @@ import { queryFieldsOf } from "@wirecat/cli-base/query";
|
|
|
4
4
|
import { Command, InvalidArgumentError } from "commander";
|
|
5
5
|
import * as v from "valibot";
|
|
6
6
|
import { answerEntity } from "../../cli/answer-entity.js";
|
|
7
|
-
import { outputFor } from "../../cli/context.js";
|
|
7
|
+
import { environmentOf, outputFor } from "../../cli/context.js";
|
|
8
8
|
import { LOCAL_READ, REMOTE_READ } from "../../cli/effects.js";
|
|
9
9
|
import { itemOf, listOf, renderAnswer, snakeKeys } from "../../cli/envelope.js";
|
|
10
10
|
import { listQuery, typedQuery, withQuery } from "../../cli/list-query.js";
|
|
@@ -74,8 +74,11 @@ const serverIssues = (server) => {
|
|
|
74
74
|
...FAILED[reason],
|
|
75
75
|
source: "server",
|
|
76
76
|
}));
|
|
77
|
-
|
|
78
|
-
?
|
|
77
|
+
const tooLarge = server.tooLarge
|
|
78
|
+
? [{ code: "outcome_unknown", reason: "search.server-too-large", source: "server" }]
|
|
79
|
+
: [];
|
|
80
|
+
return failed.length || tooLarge.length
|
|
81
|
+
? [...failed, ...tooLarge]
|
|
79
82
|
: [{ code: "outcome_unknown", reason: "search.server-capped", source: "server" }];
|
|
80
83
|
};
|
|
81
84
|
const searchCommand = (mail) => {
|
|
@@ -114,19 +117,23 @@ found_in says archive, server (new from the server) or both; meta.complete is fa
|
|
|
114
117
|
: ""}
|
|
115
118
|
Search guide: https://github.com/WireCatLabs/cli-core/blob/main/docs/search/query-language.md`);
|
|
116
119
|
if (mail.server)
|
|
117
|
-
serverSearchOptions(leaf, "the mail server (default: both)");
|
|
120
|
+
serverSearchOptions(leaf, "the mail server (default: both)", mail.serverTimeDefault);
|
|
118
121
|
leaf.action(async function (words) {
|
|
119
122
|
const options = this.opts();
|
|
120
123
|
const limit = options.limit ?? DEFAULT_LIMIT;
|
|
121
124
|
const page = options.page ?? 1;
|
|
122
125
|
const query = words.join(" ");
|
|
123
126
|
const { backend, server: bounds } = mail.server ? backendRequest(this) : {};
|
|
127
|
+
const { signal, deadline: deadlineOf } = environmentOf(this);
|
|
128
|
+
const deadline = deadlineOf?.();
|
|
124
129
|
const found = await mail.read(this, async (store, mailbox) => {
|
|
125
130
|
const step = mail.server
|
|
126
131
|
? await searchMailServer(store, mailbox, mail.server(this), {
|
|
127
132
|
text: query,
|
|
128
133
|
...(backend === undefined ? {} : { backend }),
|
|
129
134
|
...(bounds?.timeMs === undefined ? {} : { timeMs: bounds.timeMs }),
|
|
135
|
+
...(signal ? { signal } : {}),
|
|
136
|
+
...(deadline ? { deadline } : {}),
|
|
130
137
|
...(options.exact ? { exact: true } : {}),
|
|
131
138
|
...(options.timezone === undefined ? {} : { timezone: options.timezone }),
|
|
132
139
|
})
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type Deadline } from "../background/budget.js";
|
|
1
2
|
import { type Level, type PermissionKey } from "../sends/permissions.js";
|
|
2
3
|
import { type Backend, type ServerStep } from "../services/server-search.js";
|
|
3
4
|
import type { AccountKey, MessageStore } from "../store/index.js";
|
|
@@ -15,6 +16,8 @@ export interface MailServerAccess {
|
|
|
15
16
|
folders: string[];
|
|
16
17
|
close?: () => Promise<void>;
|
|
17
18
|
}>;
|
|
19
|
+
/** This mailbox's own default for the time bound, for a server whose search is slow; `--server-time` wins. */
|
|
20
|
+
timeMs?: number;
|
|
18
21
|
}
|
|
19
22
|
export interface MailServerRequest {
|
|
20
23
|
text: string;
|
|
@@ -23,6 +26,8 @@ export interface MailServerRequest {
|
|
|
23
26
|
backend?: Backend;
|
|
24
27
|
timeMs?: number;
|
|
25
28
|
signal?: AbortSignal;
|
|
29
|
+
/** The command's deadline: the time bound ends before it, so the search answers in part instead of timing out. */
|
|
30
|
+
deadline?: Deadline;
|
|
26
31
|
}
|
|
27
32
|
/**
|
|
28
33
|
* Asks the mailbox's server for candidates and saves the new ones, so the strict query that runs next on the store
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { CliError } from "@wirecat/cli-base";
|
|
2
2
|
import { importEngine } from "../attachments/extract.js";
|
|
3
|
+
import { deadlineMargin } from "../background/budget.js";
|
|
3
4
|
import { levelFor } from "../sends/permissions.js";
|
|
4
5
|
import { DEFAULT_BACKEND, SERVER_BOUNDS, SERVER_SEARCH_KEY, sourceKey, } from "../services/server-search.js";
|
|
5
6
|
import { MAX_ATTACHMENT_CHARS, readAttachments } from "./attachments.js";
|
|
@@ -38,9 +39,13 @@ export const searchMailServer = async (store, mailbox, access, request) => {
|
|
|
38
39
|
if (backend === "archive")
|
|
39
40
|
return undefined;
|
|
40
41
|
const quiet = request.backend === undefined;
|
|
41
|
-
const
|
|
42
|
-
if (!Number.isSafeInteger(
|
|
42
|
+
const asked = request.timeMs ?? access.timeMs ?? SERVER_BOUNDS.timeMs;
|
|
43
|
+
if (!Number.isSafeInteger(asked) || asked < 1 || asked > 60_000)
|
|
43
44
|
throw new CliError("validation_error", "the server time takes a whole number of milliseconds from 1 to 60000");
|
|
45
|
+
const { deadline } = request;
|
|
46
|
+
const timeMs = deadline
|
|
47
|
+
? Math.max(1, Math.min(asked, deadline.endsAt - Date.now() - deadlineMargin(deadline)))
|
|
48
|
+
: asked;
|
|
44
49
|
const report = { backend, skipped: null, calls: 0, returned: 0, new: 0, failed: [], complete: true };
|
|
45
50
|
const sources = new Map();
|
|
46
51
|
const skip = (reason) => {
|
|
@@ -77,10 +82,13 @@ export const searchMailServer = async (store, mailbox, access, request) => {
|
|
|
77
82
|
const fresh = [];
|
|
78
83
|
let rules = [];
|
|
79
84
|
const ruled = new Map();
|
|
80
|
-
|
|
85
|
+
const inputs = [];
|
|
86
|
+
const saved = [];
|
|
81
87
|
let opening;
|
|
82
88
|
let openFailure;
|
|
83
89
|
let unsupported = false;
|
|
90
|
+
const controller = new AbortController();
|
|
91
|
+
const stop = request.signal ? AbortSignal.any([request.signal, controller.signal]) : controller.signal;
|
|
84
92
|
// Opening lists the folders, a call to the server, so it runs inside the time bound too.
|
|
85
93
|
const run = async () => {
|
|
86
94
|
opening = access.open();
|
|
@@ -138,15 +146,63 @@ export const searchMailServer = async (store, mailbox, access, request) => {
|
|
|
138
146
|
sources.set(sourceKey(item.threadId, item.messageId), held ? "both" : "server");
|
|
139
147
|
if (held)
|
|
140
148
|
continue;
|
|
149
|
+
// A big message would hold the connection and the parser past the bound; sync reads it whole later.
|
|
150
|
+
if ((item.size ?? 0) > SERVER_BOUNDS.maxMessageBytes) {
|
|
151
|
+
report.tooLarge = (report.tooLarge ?? 0) + 1;
|
|
152
|
+
report.complete = false;
|
|
153
|
+
sources.delete(sourceKey(item.threadId, item.messageId));
|
|
154
|
+
continue;
|
|
155
|
+
}
|
|
141
156
|
fresh.push(item);
|
|
142
157
|
ruled.set(item, { ruleId: rule?.id ?? null, tags: tagsOf(rules, facts(item)) });
|
|
143
158
|
}
|
|
144
|
-
|
|
145
|
-
|
|
159
|
+
const raws = fresh.length && open ? await source.fetchRaw(fresh) : [];
|
|
160
|
+
// Parsing and reading attachments stay inside the bound: after it, nothing more is read or kept.
|
|
161
|
+
const now = Date.now();
|
|
162
|
+
for (const [index, item] of fresh.entries()) {
|
|
163
|
+
if (!open)
|
|
164
|
+
return;
|
|
165
|
+
const raw = raws[index];
|
|
166
|
+
try {
|
|
167
|
+
if (raw === undefined || raw instanceof Error)
|
|
168
|
+
throw raw ?? new Error("not read");
|
|
169
|
+
const mail = await parseMail(raw);
|
|
170
|
+
const files = await readAttachments(mail, item.messageId, { loadEngine: importEngine, maxAttachmentChars: MAX_ATTACHMENT_CHARS, at: now, signal: stop }, []);
|
|
171
|
+
if (!open)
|
|
172
|
+
return;
|
|
173
|
+
const folder = item.folder;
|
|
174
|
+
const validity = validities.get(item);
|
|
175
|
+
inputs.push({
|
|
176
|
+
accountId,
|
|
177
|
+
externalId: item.threadId,
|
|
178
|
+
now,
|
|
179
|
+
emails: [
|
|
180
|
+
{
|
|
181
|
+
...parsedEmail(mail, item, mailHeadersOf(raw), mailbox.account),
|
|
182
|
+
mailboxes: [
|
|
183
|
+
{ externalId: folder, name: folder, kind: null },
|
|
184
|
+
...(item.labels ?? []).map((name) => ({ externalId: `label:${name}`, name, kind: "label" })),
|
|
185
|
+
...(item.headers?.bulk ? [bulkBox] : []),
|
|
186
|
+
],
|
|
187
|
+
...(validity === undefined
|
|
188
|
+
? {}
|
|
189
|
+
: { locations: [{ folder, uidvalidity: validity, uid: Number(item.uid) }] }),
|
|
190
|
+
...(item.seen === undefined ? {} : { read: item.seen }),
|
|
191
|
+
...(item.flagged === undefined ? {} : { flagged: item.flagged }),
|
|
192
|
+
mailRuleId: ruled.get(item)?.ruleId ?? null,
|
|
193
|
+
attachments: files,
|
|
194
|
+
},
|
|
195
|
+
],
|
|
196
|
+
});
|
|
197
|
+
saved.push(item);
|
|
198
|
+
}
|
|
199
|
+
catch {
|
|
200
|
+
sources.delete(sourceKey(item.threadId, item.messageId));
|
|
201
|
+
report.complete = false;
|
|
202
|
+
}
|
|
203
|
+
}
|
|
146
204
|
};
|
|
147
|
-
const controller = new AbortController();
|
|
148
205
|
const timer = setTimeout(() => controller.abort(), timeMs);
|
|
149
|
-
const stop = request.signal ? AbortSignal.any([request.signal, controller.signal]) : controller.signal;
|
|
150
206
|
const bound = new Promise((resolve) => {
|
|
151
207
|
if (stop.aborted)
|
|
152
208
|
resolve(BOUND);
|
|
@@ -169,56 +225,18 @@ export const searchMailServer = async (store, mailbox, access, request) => {
|
|
|
169
225
|
const ended = outcome === BOUND || outcome === "done" ? "time_or_abort_bound" : outcome;
|
|
170
226
|
for (const call of pending)
|
|
171
227
|
fail(call, ended);
|
|
172
|
-
if (outcome !== "done"
|
|
228
|
+
if (outcome !== "done") {
|
|
173
229
|
// What was found but not read in time is the archive's to answer: none of it is claimed as the server's.
|
|
174
230
|
if (pending.size === 0)
|
|
175
231
|
fail(null, ended);
|
|
232
|
+
const kept = new Set(saved);
|
|
176
233
|
for (const item of fresh)
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
return { report, sources };
|
|
180
|
-
}
|
|
181
|
-
const now = Date.now();
|
|
182
|
-
const inputs = [];
|
|
183
|
-
const saved = [];
|
|
184
|
-
for (const [index, item] of fresh.entries()) {
|
|
185
|
-
const raw = raws?.[index];
|
|
186
|
-
try {
|
|
187
|
-
if (raw === undefined || raw instanceof Error)
|
|
188
|
-
throw raw ?? new Error("not read");
|
|
189
|
-
const mail = await parseMail(raw);
|
|
190
|
-
const files = await readAttachments(mail, item.messageId, { loadEngine: importEngine, maxAttachmentChars: MAX_ATTACHMENT_CHARS, at: now }, []);
|
|
191
|
-
const folder = item.folder;
|
|
192
|
-
const validity = validities.get(item);
|
|
193
|
-
inputs.push({
|
|
194
|
-
accountId,
|
|
195
|
-
externalId: item.threadId,
|
|
196
|
-
now,
|
|
197
|
-
emails: [
|
|
198
|
-
{
|
|
199
|
-
...parsedEmail(mail, item, mailHeadersOf(raw), mailbox.account),
|
|
200
|
-
mailboxes: [
|
|
201
|
-
{ externalId: folder, name: folder, kind: null },
|
|
202
|
-
...(item.labels ?? []).map((name) => ({ externalId: `label:${name}`, name, kind: "label" })),
|
|
203
|
-
...(item.headers?.bulk ? [bulkBox] : []),
|
|
204
|
-
],
|
|
205
|
-
...(validity === undefined
|
|
206
|
-
? {}
|
|
207
|
-
: { locations: [{ folder, uidvalidity: validity, uid: Number(item.uid) }] }),
|
|
208
|
-
...(item.seen === undefined ? {} : { read: item.seen }),
|
|
209
|
-
...(item.flagged === undefined ? {} : { flagged: item.flagged }),
|
|
210
|
-
mailRuleId: ruled.get(item)?.ruleId ?? null,
|
|
211
|
-
attachments: files,
|
|
212
|
-
},
|
|
213
|
-
],
|
|
214
|
-
});
|
|
215
|
-
saved.push(item);
|
|
216
|
-
}
|
|
217
|
-
catch {
|
|
218
|
-
sources.delete(sourceKey(item.threadId, item.messageId));
|
|
219
|
-
report.complete = false;
|
|
220
|
-
}
|
|
234
|
+
if (!kept.has(item))
|
|
235
|
+
sources.delete(sourceKey(item.threadId, item.messageId));
|
|
221
236
|
}
|
|
237
|
+
// The command's own deadline or a cancel ended it: its runner has answered, so nothing more is written.
|
|
238
|
+
if (request.signal?.aborted)
|
|
239
|
+
throw request.signal.reason;
|
|
222
240
|
if (inputs.length)
|
|
223
241
|
await store.mail.saveThreads(inputs);
|
|
224
242
|
// Sync never reads mail whose place it knows again, so a rule's tags go on here or never.
|
package/dist/mail/sync.js
CHANGED
|
@@ -195,13 +195,7 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
|
|
|
195
195
|
/** A quick folder's stored mail as it was before this run changed anything, with the UID of each. */
|
|
196
196
|
const folderMail = new Map();
|
|
197
197
|
const refreshed = new Map();
|
|
198
|
-
|
|
199
|
-
const holdsMail = async (folder) => {
|
|
200
|
-
holding ??= new Set((await store.mail.emailRefs({ accountId })).flatMap(({ mailboxes }) => mailboxes
|
|
201
|
-
.filter(({ kind }) => kind !== "label" && kind !== ENVELOPE && kind !== BULK)
|
|
202
|
-
.map((box) => box.externalId)));
|
|
203
|
-
return holding.has(folder);
|
|
204
|
-
};
|
|
198
|
+
const holdsMail = async (folder) => (await store.mail.emails({ accountId, mailbox: folder, limit: 1 })).length > 0;
|
|
205
199
|
for (const folder of folders) {
|
|
206
200
|
if (mboxPoint !== undefined)
|
|
207
201
|
break;
|
|
@@ -339,6 +333,9 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
|
|
|
339
333
|
for (const item of found)
|
|
340
334
|
collected.push(item);
|
|
341
335
|
}
|
|
336
|
+
// A listing that came back after the stop writes nothing: the command has answered, and the next run lists again.
|
|
337
|
+
if (signal?.aborted)
|
|
338
|
+
throw signal.reason;
|
|
342
339
|
const listed = [...new Map(collected.map((item) => [item.messageId, item])).values()].sort((a, b) => b.receivedAt.localeCompare(a.receivedAt));
|
|
343
340
|
const present = new Set(listed.map(({ messageId }) => messageId));
|
|
344
341
|
const factsFor = factsOf(listed, rules, ages);
|
|
@@ -449,7 +446,11 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
|
|
|
449
446
|
const pending = [];
|
|
450
447
|
// IMAP lists what exists now, so a mail marked deleted that it lists again is saved again, which brings it back.
|
|
451
448
|
// An archive is old: what it holds that IMAP proved deleted stays deleted.
|
|
452
|
-
const refs = await store.mail.emailRefs({
|
|
449
|
+
const refs = await store.mail.emailRefs({
|
|
450
|
+
accountId,
|
|
451
|
+
includeDeleted: snapshot,
|
|
452
|
+
externalIds: listed.map(({ messageId }) => messageId),
|
|
453
|
+
});
|
|
453
454
|
const known = new Set(refs.map(({ externalId }) => externalId));
|
|
454
455
|
const enveloped = new Map(refs.flatMap(({ externalId, mailboxes }) => {
|
|
455
456
|
const boxes = mailboxes.filter(({ kind }) => kind === ENVELOPE).map((box) => box.externalId);
|
|
@@ -712,7 +713,11 @@ const syncRound = async (source, store, { account, since, max = 200, maxBodyChar
|
|
|
712
713
|
}
|
|
713
714
|
plan?.unreadable.delete(item.messageId);
|
|
714
715
|
const header = mailHeadersOf(raws[index]);
|
|
715
|
-
|
|
716
|
+
// A stop between attachments ends the chunk; what the earlier slices saved stays.
|
|
717
|
+
const files = await readAttachments(mail, item.messageId, { loadEngine, maxAttachmentChars, at: startedAt, ...(signal ? { signal } : {}) }, attachments).catch(async (error) => {
|
|
718
|
+
await stopIfAborted();
|
|
719
|
+
throw error;
|
|
720
|
+
});
|
|
716
721
|
const input = {
|
|
717
722
|
accountId,
|
|
718
723
|
externalId: item.threadId,
|
package/dist/oauth/authorize.js
CHANGED
|
@@ -46,7 +46,7 @@ export const authorize = async (options) => {
|
|
|
46
46
|
else
|
|
47
47
|
fail(new CliError("authentication_error", `${provider.name} sign-in was declined`, { reason: "oauth.declined" }));
|
|
48
48
|
};
|
|
49
|
-
const
|
|
49
|
+
const handler = (request, response) => {
|
|
50
50
|
let url;
|
|
51
51
|
try {
|
|
52
52
|
url = new URL(request.url ?? "/", "http://127.0.0.1");
|
|
@@ -69,7 +69,11 @@ export const authorize = async (options) => {
|
|
|
69
69
|
}
|
|
70
70
|
response.end("Sign-in received; you can close this tab");
|
|
71
71
|
settle(url.searchParams);
|
|
72
|
-
}
|
|
72
|
+
};
|
|
73
|
+
const server = createServer(handler);
|
|
74
|
+
const localhost = provider.redirect.kind === "loopback" && provider.redirect.host === "localhost";
|
|
75
|
+
// `localhost` may resolve to ::1 in the browser; the IPv6 listener is best effort, as some hosts have no IPv6.
|
|
76
|
+
const server6 = localhost ? createServer(handler) : undefined;
|
|
73
77
|
const onAbort = () => fail(cancelled());
|
|
74
78
|
options.signal?.addEventListener("abort", onAbort, { once: true });
|
|
75
79
|
const timer = setTimeout(() => fail(new CliError("timeout", `${provider.name} sign-in timed out`, { reason: "oauth.timeout" })), options.timeoutMs ?? DEFAULT_TIMEOUT_MS);
|
|
@@ -81,10 +85,19 @@ export const authorize = async (options) => {
|
|
|
81
85
|
const address = server.address();
|
|
82
86
|
if (!address || typeof address === "string")
|
|
83
87
|
throw new CliError("configuration_error", "could not start the local sign-in listener");
|
|
88
|
+
if (server6) {
|
|
89
|
+
server6.on("error", () => { });
|
|
90
|
+
await new Promise((resolve) => {
|
|
91
|
+
server6.once("error", () => resolve());
|
|
92
|
+
server6.listen(address.port, "::1", () => resolve());
|
|
93
|
+
});
|
|
94
|
+
}
|
|
84
95
|
const random = randomBytes(32).toString("base64url");
|
|
85
96
|
const relay = provider.redirect.kind === "relay";
|
|
86
97
|
state = relay ? `${address.port}.${random}` : random;
|
|
87
|
-
const redirectUri = provider.redirect.kind === "relay"
|
|
98
|
+
const redirectUri = provider.redirect.kind === "relay"
|
|
99
|
+
? provider.redirect.url
|
|
100
|
+
: `http://${localhost ? "localhost" : "127.0.0.1"}:${address.port}/callback`;
|
|
88
101
|
const url = new URL(provider.authorizeUrl);
|
|
89
102
|
for (const [key, value] of Object.entries({
|
|
90
103
|
...provider.authorizeParams,
|
|
@@ -124,7 +137,10 @@ export const authorize = async (options) => {
|
|
|
124
137
|
clearTimeout(timer);
|
|
125
138
|
options.signal?.removeEventListener("abort", onAbort);
|
|
126
139
|
paste.abort();
|
|
127
|
-
server
|
|
128
|
-
|
|
140
|
+
for (const each of server6 ? [server, server6] : [server]) {
|
|
141
|
+
each.closeAllConnections();
|
|
142
|
+
if (each.listening)
|
|
143
|
+
await new Promise((resolve) => each.close(() => resolve()));
|
|
144
|
+
}
|
|
129
145
|
}
|
|
130
146
|
};
|
package/dist/oauth/provider.d.ts
CHANGED
|
@@ -39,11 +39,13 @@ export interface OAuthProvider {
|
|
|
39
39
|
revokeUrl?: string;
|
|
40
40
|
client: OAuthClient;
|
|
41
41
|
/**
|
|
42
|
-
* `loopback` sends the browser to `http://127.0.0.1:<port>/callback
|
|
42
|
+
* `loopback` sends the browser to `http://127.0.0.1:<port>/callback`, or to `http://localhost:<port>/callback` with
|
|
43
|
+
* `host: "localhost"` (the listener then takes `127.0.0.1` and `::1`). `relay` sends it to a fixed page that
|
|
43
44
|
* forwards to the loopback port, which it reads from `state` (`<port>.<random>`).
|
|
44
45
|
*/
|
|
45
46
|
redirect: {
|
|
46
47
|
kind: "loopback";
|
|
48
|
+
host?: "localhost";
|
|
47
49
|
} | {
|
|
48
50
|
kind: "relay";
|
|
49
51
|
url: string;
|
|
@@ -2,10 +2,12 @@ import type { Id } from "../domain/models.js";
|
|
|
2
2
|
import type { ServiceDeps } from "./deps.js";
|
|
3
3
|
import type { SearchQuery } from "./messages.js";
|
|
4
4
|
export declare const SERVER_SEARCH_KEY = "messages.server-search";
|
|
5
|
+
/** `maxMessageBytes`: a candidate larger than this is not downloaded; a sync reads it, and the answer says so. */
|
|
5
6
|
export declare const SERVER_BOUNDS: {
|
|
6
7
|
timeMs: number;
|
|
7
8
|
perCall: number;
|
|
8
9
|
maxCalls: number;
|
|
10
|
+
maxMessageBytes: number;
|
|
9
11
|
};
|
|
10
12
|
export declare const DEFAULT_BACKEND = "both";
|
|
11
13
|
export type Backend = "archive" | "server" | "both";
|
|
@@ -24,6 +26,8 @@ export interface ServerSearched {
|
|
|
24
26
|
chat: string | null;
|
|
25
27
|
reason: "rate_limited" | "search_failed" | "time_or_abort_bound";
|
|
26
28
|
}[];
|
|
29
|
+
/** Candidates larger than `SERVER_BOUNDS.maxMessageBytes`, left on the server unread. */
|
|
30
|
+
tooLarge?: number;
|
|
27
31
|
complete: boolean;
|
|
28
32
|
}
|
|
29
33
|
export type HitSource = "archive" | "server" | "both";
|
|
@@ -2,7 +2,8 @@ import { CliError } from "@wirecat/cli-base";
|
|
|
2
2
|
import { capability } from "../ports/messenger.js";
|
|
3
3
|
import { prepareLucene } from "./messages-search.js";
|
|
4
4
|
export const SERVER_SEARCH_KEY = "messages.server-search";
|
|
5
|
-
|
|
5
|
+
/** `maxMessageBytes`: a candidate larger than this is not downloaded; a sync reads it, and the answer says so. */
|
|
6
|
+
export const SERVER_BOUNDS = { timeMs: 5_000, perCall: 100, maxCalls: 3, maxMessageBytes: 5_000_000 };
|
|
6
7
|
export const DEFAULT_BACKEND = "both";
|
|
7
8
|
export const sourceKey = (chatId, id) => JSON.stringify([chatId, id]);
|
|
8
9
|
const merge = (one, other) => {
|
|
@@ -161,6 +161,8 @@ export interface EmailFilter extends MailFilter {
|
|
|
161
161
|
until?: number;
|
|
162
162
|
/** Only emails with an attachment whose text extraction failed with one of these errors. */
|
|
163
163
|
attachmentErrors?: string[];
|
|
164
|
+
/** Only these Message-IDs: a sync asks about what it listed, not about the whole account. */
|
|
165
|
+
externalIds?: readonly string[];
|
|
164
166
|
}
|
|
165
167
|
export interface EmailRef {
|
|
166
168
|
externalId: string;
|
|
@@ -313,6 +313,8 @@ const emailWhere = (filter) => {
|
|
|
313
313
|
if (filter.attachmentErrors !== undefined)
|
|
314
314
|
add(`e.id IN (SELECT attachable_id FROM attachments WHERE attachable_type = 'email'
|
|
315
315
|
AND extraction_error IN (${filter.attachmentErrors.map(() => "?").join(",") || "NULL"}))`, ...filter.attachmentErrors);
|
|
316
|
+
if (filter.externalIds !== undefined)
|
|
317
|
+
add("e.external_id IN (SELECT value FROM json_each(?))", JSON.stringify(filter.externalIds));
|
|
316
318
|
return [where.join(" AND "), params];
|
|
317
319
|
};
|
|
318
320
|
/** An email as a message of its thread, the shape mail had when it was stored as messages. */
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@wirecat/cli-core",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.542.0",
|
|
4
4
|
"description": "The messenger-neutral half of a messaging CLI: domain model, message locators, rendering, name resolution, the SQLite seam, the command skeleton and shared read commands, the send guard, run records and the message store",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Viacheslav Ptsarev",
|