@pmndre/pine 0.22.1 → 0.33.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. package/README.md +2 -1
  2. package/changes.json +264 -0
  3. package/docs/SKILL.md +31 -7
  4. package/docs/references/agents.md +245 -21
  5. package/docs/references/coding.md +1 -1
  6. package/docs/references/computer.md +1 -1
  7. package/docs/references/configuration.md +14 -5
  8. package/docs/references/mcp.md +9 -0
  9. package/docs/references/operations.md +21 -10
  10. package/docs/references/overview.md +3 -1
  11. package/docs/references/security.md +83 -24
  12. package/docs/references/setup.md +55 -9
  13. package/docs/references/source-types.md +1418 -0
  14. package/docs/references/sources.md +333 -19
  15. package/docs/references/stack.md +1 -0
  16. package/docs/references/tui.md +86 -12
  17. package/examples/home/AGENTS.md +4 -0
  18. package/examples/home/TOOLS.md +5 -0
  19. package/examples/home/agents/main/workspace/AGENTS.md +1 -1
  20. package/examples/home/agents/main/workspace/TOOLS.md +5 -0
  21. package/mcp/browser.d.ts +6 -0
  22. package/mcp/browser.js +19 -0
  23. package/mcp/render.d.ts +1 -0
  24. package/mcp/render.js +237 -0
  25. package/package.json +6 -4
  26. package/src/actions.d.ts +6 -1
  27. package/src/actions.js +17 -4
  28. package/src/admin.d.ts +4 -0
  29. package/src/admin.js +25 -6
  30. package/src/agents.d.ts +167 -2
  31. package/src/agents.js +912 -45
  32. package/src/catalogue.d.ts +60 -0
  33. package/src/catalogue.js +791 -0
  34. package/src/cli.js +10 -7
  35. package/src/config.d.ts +53 -4
  36. package/src/config.js +30 -5
  37. package/src/goals.d.ts +1 -1
  38. package/src/goals.js +14 -1
  39. package/src/manual.js +24 -2
  40. package/src/mime.d.ts +78 -0
  41. package/src/mime.js +342 -0
  42. package/src/net.d.ts +11 -0
  43. package/src/net.js +29 -2
  44. package/src/pine.js +10 -1
  45. package/src/router.js +3 -3
  46. package/src/sandbox-linux.d.ts +3 -1
  47. package/src/sandbox-linux.js +5 -0
  48. package/src/security.d.ts +22 -6
  49. package/src/security.js +44 -13
  50. package/src/slots.d.ts +106 -0
  51. package/src/slots.js +250 -0
  52. package/src/source-types.d.ts +20 -0
  53. package/src/sources/calendar.d.ts +24 -0
  54. package/src/sources/calendar.js +265 -0
  55. package/src/sources/command.js +109 -4
  56. package/src/sources/discord.d.ts +4 -0
  57. package/src/sources/discord.js +298 -0
  58. package/src/sources/feed.d.ts +19 -0
  59. package/src/sources/feed.js +257 -0
  60. package/src/sources/http.js +154 -103
  61. package/src/sources/mail.d.ts +51 -0
  62. package/src/sources/mail.js +1013 -0
  63. package/src/sources/matrix.d.ts +4 -0
  64. package/src/sources/matrix.js +232 -0
  65. package/src/sources/modbus.d.ts +15 -0
  66. package/src/sources/modbus.js +240 -0
  67. package/src/sources/onvif.d.ts +35 -0
  68. package/src/sources/onvif.js +326 -0
  69. package/src/sources/presence.d.ts +80 -0
  70. package/src/sources/presence.js +480 -0
  71. package/src/sources/serial.d.ts +21 -0
  72. package/src/sources/serial.js +220 -0
  73. package/src/sources/slack.d.ts +16 -0
  74. package/src/sources/slack.js +1062 -0
  75. package/src/sources/smtp.d.ts +4 -0
  76. package/src/sources/smtp.js +360 -0
  77. package/src/sources/stream.d.ts +68 -0
  78. package/src/sources/stream.js +534 -0
  79. package/src/sources/syslog.d.ts +32 -0
  80. package/src/sources/syslog.js +243 -0
  81. package/src/sources/system.d.ts +14 -0
  82. package/src/sources/system.js +236 -0
  83. package/src/sources/system.swift +102 -0
  84. package/src/sources/telegram.js +34 -1
  85. package/src/sources.d.ts +41 -1
  86. package/src/sources.js +147 -24
  87. package/src/tools.d.ts +1 -1
  88. package/src/tools.js +304 -23
  89. package/src/tui/app.d.ts +35 -2
  90. package/src/tui/app.js +306 -43
  91. package/src/tui/levels.d.ts +68 -0
  92. package/src/tui/levels.js +273 -0
  93. package/src/tui/picture.d.ts +17 -1
  94. package/src/tui/picture.js +72 -5
  95. package/src/types.d.ts +3 -0
  96. package/src/update.d.ts +9 -1
  97. package/src/update.js +75 -3
  98. package/src/watching.d.ts +156 -0
  99. package/src/watching.js +609 -4
@@ -0,0 +1,1013 @@
1
+ import crypto from "node:crypto";
2
+ import fs from "node:fs";
3
+ import net from "node:net";
4
+ import path from "node:path";
5
+ import tls from "node:tls";
6
+ import { Type } from "typebox";
7
+ import { fetch } from "undici";
8
+ import { keepFiles, parseMail, pruneFiles } from "../mime.js";
9
+ import { isLocalHost, publicOnlyAgent, reachOptions } from "../net.js";
10
+ import { errText } from "../util.js";
11
+ import { clip } from "../watching.js";
12
+ /**
13
+ * A mailbox: reports every new mail, the moment the server says there is one (IMAP IDLE).
14
+ *
15
+ * options: { host, user, password, port?, security?, mailbox?, from?, subject?, maxMb?, keepDays?,
16
+ * smtpHost?, smtpPort?, sendTo?, sender?, insecureTls?, local? }
17
+ * security "tls" (default, port 993), "starttls" (port 143), "none" only for a server on this machine or network
18
+ * mailbox which folder to watch ("INBOX")
19
+ * from only mail that says it is from one of these: an address, or "@domain"
20
+ * subject only mail whose subject this regular expression finds
21
+ * maxMb attachments are kept up to this size (10)
22
+ * sendTo addresses the agent may write to, through smtpHost. Without it nothing can be sent
23
+ * signIn "microsoft": no password. The owner signs in at Microsoft in a browser, once, with a
24
+ * code pine shows (the device code flow), and pine logs in with what it is given
25
+ * (XOAUTH2). For Outlook.com, Hotmail and Microsoft 365, which take no password over IMAP.
26
+ * Needs `clientId`: an app the owner registered at Microsoft; `tenant` says for which accounts.
27
+ *
28
+ * event "mail": { from, fromName?, to, subject, date?, text, attachments: [{ file, path?, type, size }], images?, cut? }
29
+ * It can send, to the addresses on `sendTo` only: { to, subject, text }
30
+ *
31
+ * It only reads: the mailbox is opened read-only (EXAMINE) and messages are fetched with PEEK, so
32
+ * nothing is marked as read, moved or deleted. Mail that is there when it first connects is the
33
+ * starting point and is not reported. Who a mail says it is from is what the sender wrote, and
34
+ * anyone can write any address: a mail is outside content, whoever it claims to come from.
35
+ */
36
+ /** How long things take. A test makes them short. (`second`: what a second of waiting between two questions to Microsoft lasts.) */
37
+ export const pace = { retry: 1000, retryMax: 300_000, idle: 25 * 60_000, noop: 60_000, timeout: 30_000, second: 1000 };
38
+ /**
39
+ * Where Microsoft signs people in. A test puts its stand-in here. It is not an option of the
40
+ * source: whoever names this address is handed the sign-in, so no agent and no configuration does.
41
+ */
42
+ export const oauth = { authority: "https://login.microsoftonline.com" };
43
+ /** The servers of a Microsoft mailbox, and what pine asks to be allowed there. */
44
+ const MICROSOFT = { imap: "outlook.office365.com", smtp: "smtp.office365.com", read: "https://outlook.office.com/IMAP.AccessAsUser.All", send: "https://outlook.office.com/SMTP.Send" };
45
+ const GUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
46
+ /** How many new mails one look takes; with more (a mailbox that was away for weeks), the oldest are left out. */
47
+ const MAX_CATCH_UP = 200;
48
+ const LINE_MAX = 64 * 1024;
49
+ /** Something that went wrong, in words of pine's own. What a server says about it never goes into a status line. */
50
+ class Said extends Error {
51
+ /** Trying again will not help until the owner changes something. */
52
+ fatal = false;
53
+ }
54
+ const said = (message, fatal = false) => Object.assign(new Said(message), { fatal });
55
+ const CODES = {
56
+ ECONNREFUSED: "nothing answers at that address and port",
57
+ ENOTFOUND: "that host name is not known",
58
+ EAI_AGAIN: "the host name could not be looked up",
59
+ ETIMEDOUT: "it did not answer in time",
60
+ ECONNRESET: "the connection was cut",
61
+ EHOSTUNREACH: "the host cannot be reached",
62
+ ENETUNREACH: "the network cannot be reached",
63
+ };
64
+ /** Why a connection failed, without a word the other side chose. */
65
+ function reason(e) {
66
+ if (e instanceof Said)
67
+ return e.message;
68
+ const text = errText(e);
69
+ if (/leads to an address on your own network/.test(text))
70
+ return text;
71
+ const code = String(e?.code ?? "");
72
+ if (CODES[code])
73
+ return CODES[code];
74
+ if (/CERT|SELF_SIGNED|UNABLE_TO_VERIFY|ALTNAME/.test(code))
75
+ return `its certificate could not be verified (${code})`;
76
+ return /^[A-Z_0-9]{3,40}$/.test(code) ? `the connection failed (${code})` : "the connection failed";
77
+ }
78
+ const ADDRESS = /^[^\s@<>,;"]+@[^\s@<>,;"]+\.[^\s@<>,;"]+$/;
79
+ const hostName = (v, what) => {
80
+ const host = String(v ?? "").trim();
81
+ if (!host || /[\s/:@]/.test(host.replace(/^\[.*\]$/, "x")))
82
+ throw new Error(`"${what}" is the name of a mail server alone, e.g. imap.example.com: no "imap://", no port, no path`);
83
+ return host;
84
+ };
85
+ const portOf = (v, fallback, what) => {
86
+ if (v === undefined)
87
+ return fallback;
88
+ if (!Number.isInteger(v) || v < 1 || v > 65535)
89
+ throw new Error(`"${what}" must be a port number, for example ${fallback}`);
90
+ return v;
91
+ };
92
+ export const type = {
93
+ summary: "a mailbox (IMAP): reports every new mail with its text and attachments, without marking anything as read. With \"sendTo\" it can also write to the addresses listed",
94
+ sends: 'an e-mail to an address on its "sendTo" list: {"to": "name@example.com", "subject": "...", "text": "..."}',
95
+ options: Type.Object({
96
+ host: Type.Optional(Type.String({ description: 'the IMAP server of the mailbox, e.g. imap.gmail.com, imap.mail.me.com, imap.fastmail.com. Needed, except with "signIn"' })),
97
+ user: Type.String({ description: "the user name of the mailbox, usually the e-mail address" }),
98
+ password: Type.Optional(Type.String({ description: 'the password of the mailbox: leave it out and pine asks your owner for it. Gmail, iCloud and Fastmail want an app password made for this, not the account\'s own. Not used with "signIn"', secret: true })),
99
+ signIn: Type.Optional(Type.String({ description: '"microsoft" for a mailbox at Outlook.com, Hotmail, Live or Microsoft 365, which takes no password: your owner signs in at Microsoft in their browser, once, with a code pine shows them. Needs "clientId". Leave out for every other mailbox' })),
100
+ clientId: Type.Optional(Type.String({ description: 'with "signIn": the Application (client) ID of an app your owner registered at Microsoft (entra.microsoft.com, App registrations), e.g. 11111111-2222-3333-4444-555555555555. Ask your owner for it; it is not a secret' })),
101
+ tenant: Type.Optional(Type.String({ description: 'with "signIn": whose accounts the app signs in. "common" (default): any. "consumers": private ones (Outlook.com, Hotmail). "organizations": work and school. Or one organisation\'s domain or id' })),
102
+ port: Type.Optional(Type.Number({ description: 'port of the IMAP server. Default 993 (143 with security "starttls")' })),
103
+ security: Type.Optional(Type.String({ description: '"tls" (default): encrypted from the start. "starttls": encrypted after connecting, as port 143 wants it. "none": only for a server on this computer or network' })),
104
+ mailbox: Type.Optional(Type.String({ description: 'the folder to watch, as the server names it. Default "INBOX"' })),
105
+ from: Type.Optional(Type.Array(Type.String(), { description: 'only mail that says it is from one of these: an address ("bank@example.com") or a whole domain ("@example.com"). A filter, not a proof: a sender can be forged' })),
106
+ subject: Type.Optional(Type.String({ description: 'only mail whose subject this regular expression finds, e.g. "invoice|Rechnung". Capitals do not matter' })),
107
+ maxMb: Type.Optional(Type.Number({ description: "attachments are kept up to this size, in MB. Default 10" })),
108
+ keepDays: Type.Optional(Type.Number({ description: "kept attachments are deleted after this many days. Default 7" })),
109
+ smtpHost: Type.Optional(Type.String({ description: 'for sending: the SMTP server, e.g. smtp.gmail.com. Only together with "sendTo"' })),
110
+ smtpPort: Type.Optional(Type.Number({ description: "port of the SMTP server: 465 (default, encrypted from the start) or 587 (STARTTLS)" })),
111
+ sendTo: Type.Optional(Type.Array(Type.String(), { description: 'the addresses you may write to, e.g. your owner\'s own: ["me@example.com"]. Only when your owner wants you to send mail; without it nothing can be sent' })),
112
+ sender: Type.Optional(Type.String({ description: 'the address mail is sent as. Default: "user"' })),
113
+ insecureTls: Type.Optional(Type.Boolean({ description: "accept a certificate that cannot be verified, as a server of one's own with a self-made one needs" })),
114
+ local: Type.Optional(Type.Boolean({ description: "true if the server is on your owner's own network or this computer under a name that does not say so (mail.home, a bridge on this machine). Not needed for an IP address, localhost or a .local name" })),
115
+ }, { additionalProperties: false }),
116
+ // A mailbox that is not signed in to at its provider needs its password: asked of the owner when it is left out.
117
+ needs: (o) => (o.signIn ? [] : ["password"]),
118
+ plan(o) {
119
+ const signIn = o.signIn === undefined ? undefined : String(o.signIn).trim().toLowerCase();
120
+ if (signIn !== undefined && signIn !== "microsoft")
121
+ throw new Error('"signIn" is "microsoft", for a mailbox at Outlook.com, Hotmail or Microsoft 365. Every other mailbox takes a password: leave "signIn" out');
122
+ const clientId = String(o.clientId ?? "").trim();
123
+ const tenant = String(o.tenant ?? "common").trim();
124
+ if (signIn) {
125
+ if (o.password !== undefined)
126
+ throw new Error('with "signIn" there is no password: leave "password" out');
127
+ if (!GUID.test(clientId))
128
+ throw new Error('"signIn": "microsoft" needs "clientId": the Application (client) ID of an app your owner registered at Microsoft, which looks like 11111111-2222-3333-4444-555555555555. Your owner makes one in a few minutes: the manual says how (search "sign in to an Outlook mailbox"). Ask them for the id; it is not a secret');
129
+ if (!/^[A-Za-z0-9][A-Za-z0-9.-]{0,80}$/.test(tenant))
130
+ throw new Error('"tenant" is "common", "consumers", "organizations", or the domain or id of one organisation');
131
+ }
132
+ else if (o.clientId !== undefined || o.tenant !== undefined)
133
+ throw new Error('"clientId" and "tenant" belong to "signIn": "microsoft". Leave them out for a mailbox that takes a password');
134
+ if (!signIn && !String(o.host ?? "").trim())
135
+ throw new Error('needs "host": the IMAP server of the mailbox, e.g. imap.gmail.com. (A mailbox at Outlook.com or Microsoft 365 needs "signIn": "microsoft" instead of a password, and then no host)');
136
+ const host = hostName(o.host ?? MICROSOFT.imap, "host");
137
+ const user = String(o.user ?? "").trim();
138
+ if (!user)
139
+ throw new Error(`needs "user": the user name of the mailbox, usually the e-mail address`);
140
+ const security = String(o.security ?? "tls");
141
+ if (!["tls", "starttls", "none"].includes(security))
142
+ throw new Error('"security" is "tls", "starttls" or "none"');
143
+ const local = o.local === true || isLocalHost(host);
144
+ if (security === "none" && !local)
145
+ throw new Error(`"security": "none" sends the password unencrypted, which is only taken for a server on this computer or your own network. Use "tls" (port 993) or "starttls" (port 143)`);
146
+ const port = portOf(o.port, security === "starttls" ? 143 : security === "none" ? 143 : 993, "port");
147
+ const from = (Array.isArray(o.from) ? o.from : []).map((x) => String(x).trim().toLowerCase()).filter(Boolean);
148
+ for (const f of from)
149
+ if (!(f.startsWith("@") ? /^@[^\s@]+\.[^\s@]+$/.test(f) : ADDRESS.test(f)))
150
+ throw new Error(`"${f}" in "from" is neither an address (name@example.com) nor a domain (@example.com)`);
151
+ if (o.subject !== undefined) {
152
+ try {
153
+ new RegExp(String(o.subject), "i");
154
+ }
155
+ catch (e) {
156
+ throw new Error(`"subject" is not a regular expression: ${errText(e)}`);
157
+ }
158
+ }
159
+ for (const key of ["maxMb", "keepDays"])
160
+ if (o[key] !== undefined && !(Number(o[key]) > 0))
161
+ throw new Error(`"${key}" must be a number above 0`);
162
+ if (o.maxMb !== undefined && o.maxMb > 50)
163
+ throw new Error('"maxMb" is at most 50');
164
+ const sendTo = (Array.isArray(o.sendTo) ? o.sendTo : []).map((x) => String(x).trim().toLowerCase()).filter(Boolean);
165
+ for (const a of sendTo)
166
+ if (!ADDRESS.test(a))
167
+ throw new Error(`"${a}" in "sendTo" is not an e-mail address`);
168
+ if (sendTo.length && !o.smtpHost && !signIn)
169
+ throw new Error(`to send, it needs "smtpHost": the provider's SMTP server, e.g. smtp.gmail.com`);
170
+ if (!sendTo.length && (o.smtpHost || o.smtpPort || o.sender))
171
+ throw new Error(`"smtpHost" is for sending, and sending needs "sendTo": the addresses that may be written to. Leave both out to only read mail`);
172
+ const options = { ...o, host, user, ...(from.length ? { from } : {}) };
173
+ delete options.sendTo;
174
+ if (!from.length)
175
+ delete options.from;
176
+ if (signIn) {
177
+ Object.assign(options, { signIn, clientId: clientId.toLowerCase() });
178
+ if (o.tenant !== undefined)
179
+ options.tenant = tenant;
180
+ }
181
+ const does = [
182
+ `connect to the mailbox of ${user} on ${host}:${port}${security === "none" ? " WITHOUT encryption" : ""}${local ? " (a server on your own network or this computer)" : ""} and report every new mail in ${o.mailbox ?? "INBOX"}${from.length ? ` that says it is from ${from.join(", ")}` : ""}${o.subject ? `${from.length ? " and" : ""} whose subject matches /${o.subject}/` : ""}, with its text and its attachments up to ${o.maxMb ?? 10} MB`,
183
+ ...(signIn ? [`ask you to sign in to that mailbox at Microsoft in your browser, once, with a code pine shows you (through your app ${clientId.toLowerCase()}). pine never sees your password: it keeps what Microsoft gives it for this mailbox, and renews it`] : []),
184
+ "only read: nothing is marked as read, moved or deleted",
185
+ "treat every mail as outside content: who a mail says it is from can be forged",
186
+ ];
187
+ if (sendTo.length) {
188
+ const smtpHost = hostName(o.smtpHost ?? MICROSOFT.smtp, "smtpHost");
189
+ // Microsoft takes mail on 587, encrypted after connecting.
190
+ const smtpPort = portOf(o.smtpPort, signIn ? 587 : 465, "smtpPort");
191
+ const sender = String(o.sender ?? user).trim().toLowerCase();
192
+ if (!ADDRESS.test(sender))
193
+ throw new Error(`mail is sent as "${sender}", which is not an e-mail address. Give "sender": the address to send as`);
194
+ Object.assign(options, { smtpHost, sendTo, sender, ...(signIn && o.smtpPort === undefined ? { smtpPort } : {}) });
195
+ does.push(`let the agent send e-mail as ${sender} through ${smtpHost}:${smtpPort}, to ${sendTo.join(", ")} and to nobody else`);
196
+ }
197
+ if (o.insecureTls)
198
+ does.push("without checking the server's certificate");
199
+ return {
200
+ options,
201
+ does,
202
+ opens: true,
203
+ tell: `${signIn ? "Your owner is asked next to sign in at Microsoft: pine shows them an address and a code. Until they have, the source waits. " : ""}The mail that is in the mailbox now is the starting point and is not reported; every mail that arrives from now on is. Nothing is marked as read.`,
204
+ };
205
+ },
206
+ };
207
+ /** A connection to an IMAP server: responses read whole (a literal belongs to its line), commands answered in turn. */
208
+ class Imap {
209
+ sock;
210
+ caps = new Set();
211
+ /** How often the server has said that the mailbox holds more than before. */
212
+ news = 0;
213
+ buf = Buffer.alloc(0);
214
+ lit = [];
215
+ litSize = 0;
216
+ need = 0;
217
+ cur = { text: "", literals: [] };
218
+ queue = [];
219
+ waiter;
220
+ closed;
221
+ tag = 0;
222
+ maxLiteral;
223
+ constructor(sock, maxLiteral) {
224
+ this.maxLiteral = maxLiteral;
225
+ this.attach(sock);
226
+ }
227
+ attach(sock) {
228
+ this.sock = sock;
229
+ sock.on("data", (d) => this.feed(d));
230
+ sock.on("error", (e) => this.end(e));
231
+ sock.on("close", () => this.end(said("the connection was closed")));
232
+ }
233
+ /** Let go of the socket, for it to be wrapped in TLS. */
234
+ detach() {
235
+ this.sock.removeAllListeners("data");
236
+ this.sock.removeAllListeners("error");
237
+ this.sock.removeAllListeners("close");
238
+ return this.sock;
239
+ }
240
+ end(e) {
241
+ this.closed ??= e;
242
+ this.waiter?.();
243
+ }
244
+ fail(message) {
245
+ this.end(said(message));
246
+ this.sock.destroy();
247
+ }
248
+ feed(chunk) {
249
+ if (this.need > 0) {
250
+ // Inside a literal nothing is searched: the pieces are put together once, when all are there.
251
+ this.lit.push(chunk);
252
+ this.litSize += chunk.length;
253
+ if (this.litSize < this.need)
254
+ return;
255
+ const all = Buffer.concat(this.lit);
256
+ this.cur.literals.push(all.subarray(0, this.need));
257
+ this.buf = all.subarray(this.need);
258
+ this.need = 0;
259
+ this.lit = [];
260
+ this.litSize = 0;
261
+ }
262
+ else
263
+ this.buf = this.buf.length ? Buffer.concat([this.buf, chunk]) : chunk;
264
+ this.parse();
265
+ }
266
+ parse() {
267
+ for (;;) {
268
+ const i = this.buf.indexOf("\r\n");
269
+ if (i < 0) {
270
+ if (this.buf.length > LINE_MAX)
271
+ this.fail("the server sent a line that does not end");
272
+ return;
273
+ }
274
+ if (i > LINE_MAX || this.cur.text.length > LINE_MAX * 4)
275
+ return this.fail("the server sent a line that is too long");
276
+ const line = this.buf.subarray(0, i).toString("latin1");
277
+ this.buf = this.buf.subarray(i + 2);
278
+ const m = /\{(\d+)\+?\}$/.exec(line);
279
+ if (!m) {
280
+ this.cur.text += line;
281
+ if (/^\* \d+ (EXISTS|RECENT)\b/i.test(this.cur.text))
282
+ this.news++;
283
+ this.queue.push(this.cur);
284
+ this.cur = { text: "", literals: [] };
285
+ this.waiter?.();
286
+ continue;
287
+ }
288
+ this.cur.text += line.slice(0, m.index);
289
+ const n = Number(m[1]);
290
+ if (n > this.maxLiteral || this.cur.literals.length > 32)
291
+ return this.fail("the server sent more than was asked for");
292
+ if (this.buf.length >= n) {
293
+ this.cur.literals.push(this.buf.subarray(0, n));
294
+ this.buf = this.buf.subarray(n);
295
+ continue;
296
+ }
297
+ this.need = n;
298
+ this.lit = [this.buf];
299
+ this.litSize = this.buf.length;
300
+ this.buf = Buffer.alloc(0);
301
+ return;
302
+ }
303
+ }
304
+ /** The next response. With `quiet`, nothing in time is no error: it returns undefined. */
305
+ async next(ms = pace.timeout, quiet = false) {
306
+ const until = Date.now() + ms;
307
+ while (!this.queue.length) {
308
+ if (this.closed)
309
+ throw this.closed;
310
+ const left = until - Date.now();
311
+ if (left <= 0) {
312
+ if (quiet)
313
+ return undefined;
314
+ throw said("the server did not answer in time");
315
+ }
316
+ await new Promise((resolve) => {
317
+ const timer = setTimeout(() => {
318
+ this.waiter = undefined;
319
+ resolve();
320
+ }, left);
321
+ this.waiter = () => {
322
+ clearTimeout(timer);
323
+ this.waiter = undefined;
324
+ resolve();
325
+ };
326
+ });
327
+ }
328
+ return this.queue.shift();
329
+ }
330
+ write(text) {
331
+ if (this.closed)
332
+ throw this.closed;
333
+ this.sock.write(text);
334
+ }
335
+ nextTag() {
336
+ return `p${++this.tag}`;
337
+ }
338
+ /**
339
+ * Send a command and collect what comes back until it is answered. A part that is bytes goes
340
+ * as a literal: announced, and sent when the server says to go on.
341
+ */
342
+ async cmd(parts, what, fatal = false) {
343
+ const tag = this.nextTag();
344
+ const untagged = [];
345
+ let pending = `${tag} `;
346
+ const answered = (r) => {
347
+ if (!r.text.startsWith(`${tag} `))
348
+ return void untagged.push(r);
349
+ const rest = r.text.slice(tag.length + 1);
350
+ if (/^OK\b/i.test(rest))
351
+ return { untagged, text: rest };
352
+ throw said(`the server refused to ${what}`, fatal);
353
+ };
354
+ for (const part of parts) {
355
+ if (typeof part === "string") {
356
+ pending += part;
357
+ continue;
358
+ }
359
+ this.write(`${pending}{${part.length}}\r\n`);
360
+ pending = "";
361
+ for (;;) {
362
+ const r = (await this.next());
363
+ if (r.text.startsWith("+"))
364
+ break;
365
+ const done = answered(r);
366
+ // Answered before the literal was sent: the command is over.
367
+ if (done)
368
+ return done;
369
+ }
370
+ this.write(part);
371
+ }
372
+ this.write(`${pending}\r\n`);
373
+ for (;;) {
374
+ const done = answered((await this.next()));
375
+ if (done)
376
+ return done;
377
+ }
378
+ }
379
+ /**
380
+ * Log in with what a sign-in gave instead of a password (SASL XOAUTH2). A server that refuses
381
+ * says why in a challenge first, which is answered with an empty line before the refusal comes.
382
+ */
383
+ async xoauth2(user, token) {
384
+ const tag = this.nextTag();
385
+ const words = Buffer.from(`user=${user}\x01auth=Bearer ${token}\x01\x01`).toString("base64");
386
+ // With the first line where the server takes it there, else when it says to go on.
387
+ let sent = this.caps.has("SASL-IR");
388
+ this.write(`${tag} AUTHENTICATE XOAUTH2${sent ? ` ${words}` : ""}\r\n`);
389
+ for (;;) {
390
+ const r = (await this.next());
391
+ if (r.text.startsWith("+")) {
392
+ this.write(sent ? "\r\n" : `${words}\r\n`);
393
+ sent = true;
394
+ continue;
395
+ }
396
+ if (!r.text.startsWith(`${tag} `))
397
+ continue;
398
+ const rest = r.text.slice(tag.length + 1);
399
+ if (/^OK\b/i.test(rest))
400
+ return rest;
401
+ throw said("the sign-in at Microsoft worked, but the mailbox refused it: IMAP may be switched off for this mailbox, the app may lack the permission IMAP.AccessAsUser.All, or somebody else signed in than the mailbox's owner", true);
402
+ }
403
+ }
404
+ /** What the server can do, from wherever it said so. */
405
+ learn(text) {
406
+ const m = /\bCAPABILITY\b([^\]]*)/i.exec(text);
407
+ if (!m)
408
+ return false;
409
+ this.caps = new Set(m[1].trim().toUpperCase().split(/\s+/).filter(Boolean));
410
+ return true;
411
+ }
412
+ }
413
+ /** A string as an IMAP command takes it: quoted where that can say it, else as a literal. */
414
+ const arg = (s) => (/^[\x20-\x7e]*$/.test(s) ? `"${s.replace(/[\\"]/g, "\\$&")}"` : Buffer.from(s, "utf8"));
415
+ function connect(host, port, secure, o) {
416
+ return new Promise((resolve, reject) => {
417
+ const opts = { host, port, ...reachOptions(host, o.local) };
418
+ const sock = secure ? tls.connect({ ...opts, servername: net.isIP(host) ? undefined : host, rejectUnauthorized: !o.insecure }) : net.connect(opts);
419
+ const timer = setTimeout(() => sock.destroy(said("it did not answer in time")), pace.timeout);
420
+ sock.once(secure ? "secureConnect" : "connect", () => {
421
+ clearTimeout(timer);
422
+ sock.setKeepAlive(true, 60_000);
423
+ resolve(sock);
424
+ });
425
+ sock.once("error", (e) => {
426
+ clearTimeout(timer);
427
+ reject(e);
428
+ });
429
+ });
430
+ }
431
+ /** Wrap a connection that started in the clear into TLS. */
432
+ function upgrade(plain, host, insecure) {
433
+ return new Promise((resolve, reject) => {
434
+ const sock = tls.connect({ socket: plain, servername: net.isIP(host) ? undefined : host, rejectUnauthorized: !insecure });
435
+ const timer = setTimeout(() => sock.destroy(said("it did not answer in time")), pace.timeout);
436
+ sock.once("secureConnect", () => {
437
+ clearTimeout(timer);
438
+ resolve(sock);
439
+ });
440
+ sock.once("error", (e) => {
441
+ clearTimeout(timer);
442
+ reject(e);
443
+ });
444
+ });
445
+ }
446
+ // ---- SMTP, as much of it as sending one message takes ----------------------------------------------
447
+ /** One header line's worth: nothing in it can start another header. */
448
+ const oneLine = (v) => String(v ?? "").replace(/[\r\n\x00]+/g, " ").trim();
449
+ /** A header value that is not plain ASCII, as words a mail program decodes. */
450
+ function encodeWords(text) {
451
+ if (/^[\x20-\x7e]*$/.test(text))
452
+ return text;
453
+ const words = [];
454
+ let piece = "";
455
+ for (const ch of text) {
456
+ if (Buffer.byteLength(piece + ch) > 42) {
457
+ words.push(piece);
458
+ piece = "";
459
+ }
460
+ piece += ch;
461
+ }
462
+ if (piece)
463
+ words.push(piece);
464
+ return words.map((w) => `=?utf-8?B?${Buffer.from(w).toString("base64")}?=`).join("\r\n ");
465
+ }
466
+ /** A message as it goes over the wire: headers of pine's making, the text as base64. */
467
+ export function composeMail(m) {
468
+ const domain = m.from.split("@")[1] ?? "localhost";
469
+ const body = Buffer.from(m.text.replace(/\r?\n/g, "\r\n"), "utf8").toString("base64").replace(/.{1,76}/g, "$&\r\n");
470
+ return [
471
+ `From: <${oneLine(m.from)}>`,
472
+ `To: <${oneLine(m.to)}>`,
473
+ `Subject: ${encodeWords(oneLine(m.subject).slice(0, 500))}`,
474
+ `Date: ${new Date().toUTCString().replace("GMT", "+0000")}`,
475
+ `Message-ID: <${crypto.randomBytes(12).toString("hex")}@${oneLine(domain).replace(/[^A-Za-z0-9.-]/g, "")}>`,
476
+ "MIME-Version: 1.0",
477
+ "Content-Type: text/plain; charset=utf-8",
478
+ "Content-Transfer-Encoding: base64",
479
+ "",
480
+ body,
481
+ ].join("\r\n");
482
+ }
483
+ // ---- signing in at Microsoft instead of giving a password ---------------------------------------------
484
+ /** A word the sign-in server used for what went wrong, if it is one of its plain codes. Never a sentence of its own. */
485
+ const code = (v) => (typeof v === "string" && /^[a-z_]{1,40}$/.test(v) ? v : "no reason given");
486
+ /**
487
+ * The owner signs in at Microsoft in a browser, with a code pine shows them (the device code
488
+ * flow: pine itself never sees a password, and needs no address Microsoft could call). What
489
+ * Microsoft hands over is kept in the source's own storage in pine's home, and renewed by itself.
490
+ */
491
+ function microsoft(source, o) {
492
+ const scope = [MICROSOFT.read, ...(o.send ? [MICROSOFT.send] : []), "offline_access"].join(" ");
493
+ const at = new URL(oauth.authority);
494
+ const guard = isLocalHost(at.hostname) ? undefined : publicOnlyAgent(pace.timeout);
495
+ if (guard)
496
+ source.onStop(() => guard.close());
497
+ const post = async (what, fields, signal) => {
498
+ const res = await fetch(`${oauth.authority}/${encodeURIComponent(o.tenant)}/oauth2/v2.0/${what}`, {
499
+ method: "POST",
500
+ headers: { "content-type": "application/x-www-form-urlencoded" },
501
+ body: new URLSearchParams(fields).toString(),
502
+ // What is sent here is the sign-in itself: it goes to the address it is for, and nowhere a redirect says.
503
+ redirect: "error",
504
+ signal: AbortSignal.any([signal, AbortSignal.timeout(pace.timeout)]),
505
+ ...(guard ? { dispatcher: guard } : {}),
506
+ });
507
+ return (await res.json().catch(() => ({}))) ?? {};
508
+ };
509
+ let now;
510
+ // Asked once while the source runs: a retry that follows a "no" does not put the question again.
511
+ let asked = false;
512
+ let busy;
513
+ // (Neither is registered with source.secret: both are replaced about every hour, and nothing here ever puts one into an event, a log line or a status.)
514
+ const keep = (t) => {
515
+ if (typeof t.refresh_token === "string" && t.refresh_token)
516
+ source.kv.set("signin", { refresh: t.refresh_token, scope, client: o.clientId });
517
+ now = { token: String(t.access_token), until: Date.now() + (Number(t.expires_in) > 0 ? Number(t.expires_in) : 3600) * 1000 };
518
+ return now.token;
519
+ };
520
+ const ask = async (signal) => {
521
+ if (asked)
522
+ throw said("nobody is signed in to the mailbox. To be asked again: unsubscribe and subscribe again, or restart pine", true);
523
+ asked = true;
524
+ const d = await post("devicecode", { client_id: o.clientId, scope }, signal);
525
+ if (typeof d.device_code !== "string" || typeof d.user_code !== "string" || !/^[A-Za-z0-9-]{4,24}$/.test(d.user_code)) {
526
+ throw said(`Microsoft did not start a sign-in (${code(d.error)}). Check "clientId" and "tenant", and that the app allows public client flows and personal or work accounts as the mailbox is one`, true);
527
+ }
528
+ // The page to open is Microsoft's own, whatever the answer says.
529
+ let page = "https://microsoft.com/devicelogin";
530
+ try {
531
+ const given = new URL(String(d.verification_uri));
532
+ if (given.protocol === "https:" && /(^|\.)(microsoft\.com|microsoftonline\.com|live\.com)$/.test(given.hostname))
533
+ page = given.href;
534
+ else if (given.hostname === at.hostname)
535
+ page = given.href;
536
+ }
537
+ catch {
538
+ // the usual one
539
+ }
540
+ const summary = `sign in to the mailbox ${o.user} at Microsoft\nOpen ${page} in your browser, enter the code ${d.user_code}, and sign in as ${o.user}. pine carries on by itself once that is done.`;
541
+ o.say(`waiting for you to sign in: open ${page} and enter the code ${d.user_code}`);
542
+ o.waiting();
543
+ let given;
544
+ // The dialog only shows the code. What counts is what Microsoft says; a "no" gives up, and no answer at all changes nothing.
545
+ void source.approvals.request(summary, "this mailbox takes no password: Microsoft asks you yourself and gives pine a sign-in that it renews. pine does not see your password. Answer no to give up", []).then((a) => {
546
+ if (!a.allow && a.by !== "timeout")
547
+ given = false;
548
+ });
549
+ const close = (allow) => {
550
+ const open = source.approvals.pending().find((r) => r.summary === summary);
551
+ if (open)
552
+ source.approvals.decide(open.id, allow, "pine");
553
+ };
554
+ let every = Math.max(1, Number(d.interval) || 5);
555
+ const until = Date.now() + Math.min(Number(d.expires_in) > 0 ? Number(d.expires_in) : 900, 1800) * pace.second;
556
+ try {
557
+ while (Date.now() < until) {
558
+ await new Promise((resolve) => {
559
+ const done = () => (clearTimeout(timer), signal.removeEventListener("abort", done), resolve());
560
+ const timer = setTimeout(done, every * pace.second);
561
+ signal.addEventListener("abort", done, { once: true });
562
+ });
563
+ if (signal.aborted)
564
+ throw said("stopped");
565
+ if (given === false)
566
+ throw said("the sign-in was not made: you answered no. To be asked again: unsubscribe and subscribe again, or restart pine", true);
567
+ const t = await post("token", { grant_type: "urn:ietf:params:oauth:grant-type:device_code", client_id: o.clientId, device_code: d.device_code }, signal);
568
+ if (typeof t.access_token === "string" && t.access_token) {
569
+ if (typeof t.refresh_token !== "string" || !t.refresh_token)
570
+ source.log.warn('Microsoft gave no way to renew the sign-in: it will have to be made again in about an hour. The app lacks "offline_access"');
571
+ // Signed in: should Microsoft one day take it back, the owner is asked again. Only a "no" is not asked twice.
572
+ asked = false;
573
+ return keep(t);
574
+ }
575
+ if (t.error === "authorization_pending")
576
+ continue;
577
+ if (t.error === "slow_down")
578
+ every += 5;
579
+ else
580
+ throw said(t.error === "authorization_declined" ? "the sign-in was declined at Microsoft" : t.error === "expired_token" ? "the code ran out before anybody signed in" : `Microsoft refused the sign-in (${code(t.error)})`, true);
581
+ }
582
+ throw said("the code ran out before anybody signed in. To be asked again: unsubscribe and subscribe again, or restart pine", true);
583
+ }
584
+ finally {
585
+ close(now !== undefined);
586
+ }
587
+ };
588
+ const fresh = async (signal) => {
589
+ if (now && now.until - Date.now() > 120_000)
590
+ return now.token;
591
+ now = undefined;
592
+ const kept = source.kv.get("signin");
593
+ // What was given for other rights, or to another app, is not what is needed now.
594
+ if (kept?.refresh && kept.scope === scope && kept.client === o.clientId) {
595
+ const t = await post("token", { grant_type: "refresh_token", client_id: o.clientId, refresh_token: kept.refresh, scope }, signal);
596
+ if (typeof t.access_token === "string" && t.access_token)
597
+ return keep(t);
598
+ // Only "this is no good any more" lets go of it. Microsoft being unwell does not.
599
+ if (t.error !== "invalid_grant" && t.error !== "interaction_required")
600
+ throw said(`Microsoft did not renew the sign-in (${code(t.error)})`);
601
+ source.kv.set("signin", null);
602
+ source.log.info("the sign-in is no longer good (signed out, a new password, or unused for too long)");
603
+ }
604
+ return ask(signal);
605
+ };
606
+ return {
607
+ // One at a time: IMAP and a mail being sent must not both ask the owner.
608
+ token: (signal) => (busy ??= fresh(signal).finally(() => (busy = undefined))),
609
+ spent: () => void (now = undefined),
610
+ };
611
+ }
612
+ async function smtpSend(o) {
613
+ const secure = o.port === 465;
614
+ let sock = await connect(o.host, o.port, secure, o);
615
+ let encrypted = secure;
616
+ try {
617
+ let buffer = "";
618
+ let waiter;
619
+ let closed;
620
+ const listen = (s) => {
621
+ s.on("data", (d) => {
622
+ buffer += d.toString("latin1");
623
+ if (buffer.length > LINE_MAX * 4)
624
+ s.destroy();
625
+ waiter?.();
626
+ });
627
+ s.on("error", (e) => ((closed ??= e), waiter?.()));
628
+ s.on("close", () => ((closed ??= said("the mail server closed the connection")), waiter?.()));
629
+ };
630
+ listen(sock);
631
+ /** One reply, which may be several lines ("250-..." and at last "250 ..."). */
632
+ const reply = async () => {
633
+ const until = Date.now() + pace.timeout;
634
+ for (;;) {
635
+ const lines = buffer.split("\r\n");
636
+ const last = lines.findIndex((l) => /^\d{3}(?: |$)/.test(l));
637
+ if (last >= 0 && last < lines.length - 1) {
638
+ buffer = lines.slice(last + 1).join("\r\n");
639
+ return { code: Number(lines[last].slice(0, 3)), lines: lines.slice(0, last + 1).map((l) => l.slice(4)) };
640
+ }
641
+ if (closed)
642
+ throw closed;
643
+ const left = until - Date.now();
644
+ if (left <= 0)
645
+ throw said("the mail server did not answer in time");
646
+ await new Promise((resolve) => {
647
+ const timer = setTimeout(resolve, left);
648
+ waiter = () => {
649
+ clearTimeout(timer);
650
+ waiter = undefined;
651
+ resolve();
652
+ };
653
+ });
654
+ }
655
+ };
656
+ const ask = async (line, want, what) => {
657
+ sock.write(`${line}\r\n`);
658
+ const r = await reply();
659
+ if (!want.includes(r.code))
660
+ throw said(`the mail server refused ${what} (${r.code})`);
661
+ return r;
662
+ };
663
+ const hello = async () => (await ask("EHLO pine.localhost", [250], "the greeting")).lines.map((l) => l.toUpperCase());
664
+ const greeting = await reply();
665
+ if (greeting.code !== 220)
666
+ throw said(`the mail server is not ready (${greeting.code})`);
667
+ let caps = await hello();
668
+ if (!encrypted && caps.some((c) => c === "STARTTLS")) {
669
+ await ask("STARTTLS", [220], "to encrypt the connection");
670
+ sock.removeAllListeners("data");
671
+ sock.removeAllListeners("error");
672
+ sock.removeAllListeners("close");
673
+ buffer = "";
674
+ sock = await upgrade(sock, o.host, o.insecure);
675
+ listen(sock);
676
+ encrypted = true;
677
+ caps = await hello();
678
+ }
679
+ // A password is never sent in the clear to a server out on the internet.
680
+ if (!encrypted && !o.local)
681
+ throw said("the mail server offers no encryption on this port. Use port 465, or 587 on a server that has STARTTLS");
682
+ const auth = caps.find((c) => c.startsWith("AUTH"))?.split(/[ =]+/).slice(1) ?? [];
683
+ if (o.token) {
684
+ if (!auth.includes("XOAUTH2"))
685
+ throw said("the mail server takes no sign-in of this kind (XOAUTH2) on this port");
686
+ sock.write(`AUTH XOAUTH2 ${Buffer.from(`user=${o.user}\x01auth=Bearer ${o.token}\x01\x01`).toString("base64")}\r\n`);
687
+ let r = await reply();
688
+ // A refusal comes as a challenge first, answered with an empty line.
689
+ if (r.code === 334) {
690
+ sock.write("\r\n");
691
+ r = await reply();
692
+ }
693
+ if (r.code !== 235)
694
+ throw said(`the mail server refused the sign-in (${r.code}): sending may be switched off for this mailbox ("Authenticated SMTP"), or the app lacks the permission SMTP.Send`);
695
+ }
696
+ else if (auth.includes("PLAIN"))
697
+ await ask(`AUTH PLAIN ${Buffer.from(`\0${o.user}\0${o.password}`).toString("base64")}`, [235], "the login: check the user name and the password");
698
+ else if (auth.includes("LOGIN")) {
699
+ await ask("AUTH LOGIN", [334], "the login");
700
+ await ask(Buffer.from(o.user).toString("base64"), [334], "the login");
701
+ await ask(Buffer.from(o.password).toString("base64"), [235], "the login: check the user name and the password");
702
+ }
703
+ else if (auth.length || !o.local)
704
+ throw said("the mail server offers no way to log in with a password");
705
+ await ask(`MAIL FROM:<${o.from}>`, [250], "the sender");
706
+ await ask(`RCPT TO:<${o.to}>`, [250, 251], "the recipient");
707
+ await ask("DATA", [354], "the message");
708
+ // A line that begins with a dot gets a second one, so that none of them ends the message early.
709
+ await ask(`${o.message.replace(/\r?\n/g, "\r\n").replace(/^\./gm, "..")}\r\n.`, [250], "the message");
710
+ sock.write("QUIT\r\n");
711
+ }
712
+ finally {
713
+ sock.destroy();
714
+ }
715
+ }
716
+ // ---- the source ---------------------------------------------------------------------------------
717
+ const clock = () => new Date().toTimeString().slice(0, 8);
718
+ export default async function mail(source) {
719
+ const o = source.options;
720
+ const signIn = o.signIn === "microsoft";
721
+ const host = String(o.host ?? (signIn ? MICROSOFT.imap : ""));
722
+ const user = String(o.user ?? "");
723
+ const password = String(o.password ?? "");
724
+ if (!host || !user || (!signIn && !password))
725
+ throw new Error(`needs "host", "user" and "password": the IMAP server and the login of the mailbox`);
726
+ if (signIn && !GUID.test(String(o.clientId ?? "")))
727
+ throw new Error('"signIn": "microsoft" needs "clientId": the Application (client) ID of the app registered at Microsoft');
728
+ source.secret(password || undefined);
729
+ const security = String(o.security ?? "tls");
730
+ const local = o.local === true || isLocalHost(host);
731
+ if (security === "none" && !local)
732
+ throw new Error(`"security": "none" is only for a server on this computer or your own network`);
733
+ const port = Number(o.port ?? (security === "tls" ? 993 : 143));
734
+ const insecure = o.insecureTls === true;
735
+ const mailbox = String(o.mailbox ?? "INBOX");
736
+ const from = (Array.isArray(o.from) ? o.from : []).map((x) => String(x).toLowerCase());
737
+ const subject = o.subject ? new RegExp(String(o.subject), "i") : undefined;
738
+ const maxBytes = Math.min(Number(o.maxMb ?? 10), 50) * 1024 * 1024;
739
+ // What is fetched of one message: enough for one attachment of the largest size that is kept, as base64.
740
+ const fetchBytes = Math.ceil(maxBytes * 1.4) + 2 * 1024 * 1024;
741
+ const keepMs = Number(o.keepDays ?? 7) * 86_400_000;
742
+ const dir = path.join(source.dataDir, "files");
743
+ fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
744
+ source.media(dir);
745
+ pruneFiles(dir, keepMs);
746
+ const pruner = setInterval(() => pruneFiles(dir, keepMs), 6 * 3_600_000);
747
+ pruner.unref?.();
748
+ let stopped = false;
749
+ let current;
750
+ let wake;
751
+ let reported = 0;
752
+ let left = 0;
753
+ const say = (line) => source.status(`${line}${reported || left ? `. ${reported} reported${left ? `, ${left} left out by the filters` : ""}` : ""}`);
754
+ // Aborted when the source stops: a question to Microsoft that is under way ends with it.
755
+ const halt = new AbortController();
756
+ const sendTo = (Array.isArray(o.sendTo) ? o.sendTo : []).map((x) => String(x).trim().toLowerCase());
757
+ // Called when the owner has been asked to sign in: the source counts as started and waits, so
758
+ // that neither the agent that set it up nor pine's own start hangs on somebody opening a browser.
759
+ let begun;
760
+ const signedIn = signIn ? microsoft(source, { user, clientId: String(o.clientId).toLowerCase(), tenant: String(o.tenant ?? "common"), send: sendTo.length > 0, say, waiting: () => begun?.() }) : undefined;
761
+ const wanted = (m) => (!from.length || from.some((f) => (f.startsWith("@") ? m.from.endsWith(f) : m.from === f))) && (!subject || subject.test(m.subject));
762
+ const take = (raw, size) => {
763
+ const m = parseMail(raw, { attachmentBytes: maxBytes });
764
+ // Only the beginning of a very large message is fetched: what its last file holds is not all there.
765
+ const cut = size > raw.length;
766
+ const lastFile = m.attachments.at(-1);
767
+ if (cut && lastFile)
768
+ Object.assign(lastFile, { skipped: true, bytes: Buffer.alloc(0) });
769
+ if (!wanted(m))
770
+ return void left++;
771
+ const kept = keepFiles(dir, m);
772
+ reported++;
773
+ source.emit("mail", { from: m.from, ...(m.fromName ? { fromName: clip(m.fromName, 200) } : {}), to: m.to.slice(0, 20), subject: clip(m.subject, 300), ...(m.date ? { date: m.date } : {}), text: clip(m.text, 6000), attachments: kept.attachments, ...(kept.images ? { images: kept.images } : {}), ...(cut ? { cut: true } : {}) });
774
+ };
775
+ /** One connection, from the greeting until it breaks. `ready` is called when it is logged in and watching. */
776
+ const session = async (ready) => {
777
+ const sock = await connect(host, port, security === "tls", { local, insecure });
778
+ current = sock;
779
+ const imap = new Imap(sock, fetchBytes + 4096);
780
+ try {
781
+ if (stopped)
782
+ return;
783
+ const greeting = (await imap.next());
784
+ if (!/^\* (OK|PREAUTH)\b/i.test(greeting.text))
785
+ throw said("the server turned the connection away");
786
+ const capabilities = async () => {
787
+ const r = await imap.cmd(["CAPABILITY"], "say what it can do");
788
+ for (const u of r.untagged)
789
+ imap.learn(u.text);
790
+ };
791
+ if (!imap.learn(greeting.text))
792
+ await capabilities();
793
+ if (security === "starttls") {
794
+ if (!imap.caps.has("STARTTLS"))
795
+ throw said('the server offers no STARTTLS on this port. Use "security": "tls" with port 993', true);
796
+ await imap.cmd(["STARTTLS"], "encrypt the connection");
797
+ const secure = await upgrade(imap.detach(), host, insecure);
798
+ current = secure;
799
+ imap.attach(secure);
800
+ await capabilities();
801
+ }
802
+ if (signedIn) {
803
+ if (!imap.caps.has("AUTH=XOAUTH2"))
804
+ throw said("the server takes no sign-in of this kind (XOAUTH2)", true);
805
+ const token = await signedIn.token(halt.signal);
806
+ if (stopped)
807
+ return;
808
+ let login;
809
+ try {
810
+ login = await imap.xoauth2(user, token);
811
+ }
812
+ catch (e) {
813
+ signedIn.spent();
814
+ throw e;
815
+ }
816
+ if (!imap.learn(login))
817
+ await capabilities();
818
+ }
819
+ else if (!/^\* PREAUTH\b/i.test(greeting.text)) {
820
+ if (imap.caps.has("LOGINDISABLED"))
821
+ throw said("the server takes no login with a password on this connection", true);
822
+ const login = await imap.cmd(["LOGIN ", arg(user), " ", arg(password)], "log in: check the user name and the password (Gmail, iCloud and Fastmail want an app password)", true);
823
+ if (!imap.learn(login.text))
824
+ await capabilities();
825
+ }
826
+ const opened = await imap.cmd(["EXAMINE ", arg(mailbox)], `open the mailbox "${clip(mailbox, 60)}"`, true);
827
+ const facts = [...opened.untagged.map((u) => u.text), opened.text].join("\n");
828
+ const validity = Number(/\[UIDVALIDITY (\d+)\]/i.exec(facts)?.[1] ?? 0);
829
+ const uidNext = /\[UIDNEXT (\d+)\]/i.exec(facts)?.[1];
830
+ let at = source.kv.get("at");
831
+ if (!at || at.validity !== validity) {
832
+ // The starting point: what is there now is not news. (Also when the server numbered its mail anew.)
833
+ let last = uidNext ? Number(uidNext) - 1 : 0;
834
+ if (!uidNext) {
835
+ const all = await imap.cmd(["UID SEARCH ALL"], "list the mailbox");
836
+ last = Math.max(0, ...all.untagged.flatMap((u) => (/^\* SEARCH\b/i.test(u.text) ? u.text.split(/\s+/).slice(2).map(Number).filter(Number.isFinite) : [])));
837
+ }
838
+ at = { validity, last };
839
+ source.kv.set("at", at);
840
+ }
841
+ say(`connected to ${host} as ${user}, waiting for mail in ${clip(mailbox, 60)}`);
842
+ ready();
843
+ const catchUp = async () => {
844
+ const found = await imap.cmd([`UID SEARCH UID ${at.last + 1}:*`], "look for new mail");
845
+ // "n:*" always names the newest mail, also when it is older than n.
846
+ let uids = [...new Set(found.untagged.flatMap((u) => (/^\* SEARCH\b/i.test(u.text) ? u.text.split(/\s+/).slice(2).map(Number) : [])))].filter((n) => Number.isInteger(n) && n > at.last).sort((a, b) => a - b);
847
+ if (uids.length > MAX_CATCH_UP) {
848
+ source.log.warn(`${uids.length} new mails at once: the oldest ${uids.length - MAX_CATCH_UP} are left out`);
849
+ uids = uids.slice(-MAX_CATCH_UP);
850
+ }
851
+ for (const uid of uids) {
852
+ if (stopped)
853
+ return;
854
+ const got = await imap.cmd([`UID FETCH ${uid} (UID RFC822.SIZE BODY.PEEK[]<0.${fetchBytes}>)`], "hand over a mail");
855
+ const fetched = got.untagged.find((u) => /^\* \d+ FETCH\b/i.test(u.text) && u.literals.length && new RegExp(`\\bUID ${uid}\\b`, "i").test(u.text));
856
+ if (fetched) {
857
+ const raw = fetched.literals.reduce((a, b) => (b.length > a.length ? b : a));
858
+ try {
859
+ take(raw, Number(/RFC822\.SIZE (\d+)/i.exec(fetched.text)?.[1] ?? raw.length));
860
+ }
861
+ catch (e) {
862
+ source.log.warn(`could not read mail ${uid}`, errText(e));
863
+ }
864
+ }
865
+ // Remembered mail by mail: a connection that breaks half way repeats nothing.
866
+ at = { validity, last: uid };
867
+ source.kv.set("at", at);
868
+ }
869
+ if (uids.length)
870
+ say(`connected to ${host} as ${user}, waiting for mail in ${clip(mailbox, 60)}`);
871
+ };
872
+ let idle = imap.caps.has("IDLE");
873
+ for (;;) {
874
+ const before = imap.news;
875
+ await catchUp();
876
+ if (stopped)
877
+ return;
878
+ // A mail that arrived while the last ones were fetched was announced then, and will not be again.
879
+ if (imap.news !== before)
880
+ continue;
881
+ if (!idle) {
882
+ // A server that cannot tell is asked once a minute.
883
+ const until = Date.now() + pace.noop;
884
+ while (Date.now() < until)
885
+ if (!(await imap.next(until - Date.now(), true)))
886
+ break;
887
+ await imap.cmd(["NOOP"], "answer");
888
+ continue;
889
+ }
890
+ const tag = imap.nextTag();
891
+ imap.write(`${tag} IDLE\r\n`);
892
+ let going = false;
893
+ let over = false;
894
+ const until = Date.now() + pace.idle;
895
+ for (;;) {
896
+ // Until it says "go on", an answer is due; after that, silence is what waiting looks like.
897
+ const r = await imap.next(going ? Math.max(1, until - Date.now()) : pace.timeout, going);
898
+ if (!r)
899
+ break;
900
+ if (r.text.startsWith("+"))
901
+ going = true;
902
+ else if (r.text.startsWith(`${tag} `)) {
903
+ // Refused: this server only says it can.
904
+ over = true;
905
+ idle = false;
906
+ break;
907
+ }
908
+ else if (/^\* \d+ (EXISTS|RECENT)\b/i.test(r.text))
909
+ break;
910
+ else if (/^\* BYE\b/i.test(r.text))
911
+ throw said("the server ended the connection");
912
+ }
913
+ if (over)
914
+ continue;
915
+ imap.write("DONE\r\n");
916
+ for (;;) {
917
+ const r = (await imap.next());
918
+ if (r.text.startsWith(`${tag} `))
919
+ break;
920
+ }
921
+ }
922
+ }
923
+ finally {
924
+ current = undefined;
925
+ imap.sock.destroy();
926
+ sock.destroy();
927
+ }
928
+ };
929
+ source.onStop(() => {
930
+ stopped = true;
931
+ halt.abort();
932
+ clearInterval(pruner);
933
+ current?.destroy();
934
+ wake?.();
935
+ });
936
+ // The first connection says whether the login works; a wrong password is the reason the source is
937
+ // not running. A server that cannot be reached right now is tried again and again instead.
938
+ await new Promise((resolve, reject) => {
939
+ let first = true;
940
+ const settle = (e) => {
941
+ if (!first)
942
+ return;
943
+ first = false;
944
+ if (e)
945
+ reject(e);
946
+ else
947
+ resolve();
948
+ };
949
+ begun = () => settle();
950
+ void (async () => {
951
+ let pause = pace.retry;
952
+ let failing = false;
953
+ while (!stopped) {
954
+ try {
955
+ await session(() => {
956
+ pause = pace.retry;
957
+ if (failing)
958
+ source.log.info("connected again");
959
+ failing = false;
960
+ settle();
961
+ });
962
+ }
963
+ catch (e) {
964
+ if (stopped)
965
+ break;
966
+ const why = reason(e);
967
+ if (first && e instanceof Said && e.fatal)
968
+ return settle(new Error(why));
969
+ // Not signed in and not to be asked again: nothing to try until the owner does something.
970
+ if (e instanceof Said && e.fatal && signedIn)
971
+ pause = pace.retryMax;
972
+ if (!failing)
973
+ source.log.warn(`not connected: ${why}`);
974
+ failing = true;
975
+ say(`not connected since ${clock()}: ${why}. Trying again`);
976
+ }
977
+ settle();
978
+ if (stopped)
979
+ break;
980
+ await new Promise((r) => {
981
+ const timer = setTimeout(r, pause);
982
+ wake = () => (clearTimeout(timer), r());
983
+ });
984
+ wake = undefined;
985
+ pause = Math.min(pause * 2, pace.retryMax);
986
+ }
987
+ settle();
988
+ })();
989
+ });
990
+ // Sending: to the addresses the owner put on the list, and to nobody else.
991
+ const smtpHost = typeof o.smtpHost === "string" ? o.smtpHost : signIn ? MICROSOFT.smtp : "";
992
+ const sender = String(o.sender ?? user);
993
+ source.onSend(async (m) => {
994
+ if (!smtpHost || !sendTo.length)
995
+ throw new Error("this mailbox was set up to read only: nothing can be sent through it");
996
+ const to = String(m.to ?? (sendTo.length === 1 ? sendTo[0] : "")).trim().toLowerCase();
997
+ if (!to)
998
+ throw new Error(`say whom to write to: {"to": "...", "subject": "...", "text": "..."}. It may write to ${sendTo.join(", ")}`);
999
+ if (!sendTo.includes(to))
1000
+ throw new Error(`"${clip(to.replace(/[\r\n]+/g, " "), 100)}" is not an address this source may write to (${sendTo.join(", ")})`);
1001
+ const text = String(m.text ?? "");
1002
+ if (!text.trim())
1003
+ throw new Error('there is nothing to send: give "text"');
1004
+ const title = oneLine(m.subject) || "A message from your agent";
1005
+ try {
1006
+ await smtpSend({ host: smtpHost, port: Number(o.smtpPort ?? (signIn ? 587 : 465)), user, password, token: signedIn ? await signedIn.token(halt.signal) : undefined, local: o.local === true || isLocalHost(smtpHost), insecure, from: sender, to, message: composeMail({ from: sender, to, subject: title, text }) });
1007
+ }
1008
+ catch (e) {
1009
+ throw new Error(`the mail was not sent: ${reason(e)}`);
1010
+ }
1011
+ return `sent to ${to}`;
1012
+ });
1013
+ }