@pmndre/pine 0.22.1 → 0.33.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. package/README.md +2 -1
  2. package/changes.json +265 -0
  3. package/docs/SKILL.md +31 -7
  4. package/docs/references/agents.md +245 -21
  5. package/docs/references/coding.md +1 -1
  6. package/docs/references/computer.md +1 -1
  7. package/docs/references/configuration.md +14 -5
  8. package/docs/references/mcp.md +9 -0
  9. package/docs/references/operations.md +20 -9
  10. package/docs/references/overview.md +3 -1
  11. package/docs/references/security.md +83 -24
  12. package/docs/references/setup.md +55 -9
  13. package/docs/references/source-types.md +1418 -0
  14. package/docs/references/sources.md +333 -19
  15. package/docs/references/stack.md +1 -0
  16. package/docs/references/tui.md +86 -12
  17. package/examples/home/AGENTS.md +4 -0
  18. package/examples/home/TOOLS.md +5 -0
  19. package/examples/home/agents/main/workspace/AGENTS.md +1 -1
  20. package/examples/home/agents/main/workspace/TOOLS.md +5 -0
  21. package/mcp/browser.d.ts +6 -0
  22. package/mcp/browser.js +19 -0
  23. package/mcp/render.d.ts +1 -0
  24. package/mcp/render.js +237 -0
  25. package/package.json +3 -1
  26. package/src/actions.d.ts +6 -1
  27. package/src/actions.js +17 -4
  28. package/src/admin.d.ts +4 -0
  29. package/src/admin.js +25 -6
  30. package/src/agents.d.ts +167 -2
  31. package/src/agents.js +912 -45
  32. package/src/catalogue.d.ts +60 -0
  33. package/src/catalogue.js +791 -0
  34. package/src/cli.js +10 -7
  35. package/src/config.d.ts +53 -4
  36. package/src/config.js +30 -5
  37. package/src/goals.d.ts +1 -1
  38. package/src/goals.js +14 -1
  39. package/src/manual.js +24 -2
  40. package/src/mime.d.ts +78 -0
  41. package/src/mime.js +342 -0
  42. package/src/net.d.ts +11 -0
  43. package/src/net.js +29 -2
  44. package/src/pine.js +10 -1
  45. package/src/router.js +3 -3
  46. package/src/sandbox-linux.d.ts +3 -1
  47. package/src/sandbox-linux.js +5 -0
  48. package/src/security.d.ts +22 -6
  49. package/src/security.js +44 -13
  50. package/src/slots.d.ts +106 -0
  51. package/src/slots.js +250 -0
  52. package/src/source-types.d.ts +20 -0
  53. package/src/sources/calendar.d.ts +24 -0
  54. package/src/sources/calendar.js +265 -0
  55. package/src/sources/command.js +109 -4
  56. package/src/sources/discord.d.ts +4 -0
  57. package/src/sources/discord.js +298 -0
  58. package/src/sources/feed.d.ts +19 -0
  59. package/src/sources/feed.js +257 -0
  60. package/src/sources/http.js +154 -103
  61. package/src/sources/mail.d.ts +51 -0
  62. package/src/sources/mail.js +1013 -0
  63. package/src/sources/matrix.d.ts +4 -0
  64. package/src/sources/matrix.js +232 -0
  65. package/src/sources/modbus.d.ts +15 -0
  66. package/src/sources/modbus.js +240 -0
  67. package/src/sources/onvif.d.ts +35 -0
  68. package/src/sources/onvif.js +326 -0
  69. package/src/sources/presence.d.ts +80 -0
  70. package/src/sources/presence.js +480 -0
  71. package/src/sources/serial.d.ts +21 -0
  72. package/src/sources/serial.js +220 -0
  73. package/src/sources/slack.d.ts +16 -0
  74. package/src/sources/slack.js +1062 -0
  75. package/src/sources/smtp.d.ts +4 -0
  76. package/src/sources/smtp.js +360 -0
  77. package/src/sources/stream.d.ts +68 -0
  78. package/src/sources/stream.js +534 -0
  79. package/src/sources/syslog.d.ts +32 -0
  80. package/src/sources/syslog.js +243 -0
  81. package/src/sources/system.d.ts +14 -0
  82. package/src/sources/system.js +236 -0
  83. package/src/sources/system.swift +102 -0
  84. package/src/sources/telegram.js +34 -1
  85. package/src/sources.d.ts +41 -1
  86. package/src/sources.js +147 -24
  87. package/src/tools.d.ts +1 -1
  88. package/src/tools.js +304 -23
  89. package/src/tui/app.d.ts +35 -2
  90. package/src/tui/app.js +306 -43
  91. package/src/tui/levels.d.ts +68 -0
  92. package/src/tui/levels.js +273 -0
  93. package/src/tui/picture.d.ts +17 -1
  94. package/src/tui/picture.js +72 -5
  95. package/src/types.d.ts +3 -0
  96. package/src/update.d.ts +9 -1
  97. package/src/update.js +75 -3
  98. package/src/watching.d.ts +156 -0
  99. package/src/watching.js +609 -4
@@ -1,134 +1,136 @@
1
- import net from "node:net";
1
+ import fs from "node:fs";
2
+ import path from "node:path";
2
3
  import { Type } from "typebox";
3
- import { fetch } from "undici";
4
- import { isLocalName, isPrivateAddress, publicOnlyAgent } from "../net.js";
4
+ import { localTarget } from "../net.js";
5
5
  import { errText, getPath, parseDuration } from "../util.js";
6
- import { clip, found, harmless, lineDiff, pageText, pattern, shown } from "../watching.js";
6
+ import { ask, askable, clip, found, guardFor, harmless, keepSince, lineDiff, numbers, pageText, pattern, rendered, selected, selector, shown, sinceOf, whyFailed } from "../watching.js";
7
7
  /**
8
8
  * A web address, looked at again and again: reports when what it answers changes, and when it
9
9
  * stops or starts answering. For pages and services that cannot call you: a price, a version, a
10
10
  * status page, a device with a state page on your network.
11
11
  *
12
- * options: { url, every?, path?, match?, headers?, token?, tokenHeader?, local? }
12
+ * options: { url, every?, path?, values?, match?, select?, attr?, render?, method?, body?, headers?, token?, tokenHeader?, local? }
13
13
  * every how often to look ("5m" by default)
14
14
  * path for a JSON answer: watch only this field ("data.price")
15
+ * values for a JSON answer: watch several fields, each under a name ({ "price": "data.price" })
15
16
  * match for a page or text: watch only what this regular expression finds
17
+ * select for a page: watch only the text of the first element this CSS selector matches
18
+ * attr with select: the value of this attribute of the element instead of its text
19
+ * render for a page that a script fills in after loading: every look opens the page in pine's browser,
20
+ * without a window, and takes what a reader sees then. `select` is the browser's own CSS then
21
+ * method, body "POST" and what to send, for an address that is asked that way (GraphQL)
16
22
  * headers headers to send
17
23
  * token sent as "Authorization: Bearer <token>", or as it is in the header `tokenHeader`
18
24
  * local the address is on the owner's own network, under a name that does not say so
19
25
  *
20
- * event "changed": { url, value, previous } with path or match
21
- * { url, added, removed, more? } without: the lines that are new and gone
26
+ * event "changed": { url, value, previous, number?, previousNumber? } with path, match or select
27
+ * { url, values, previous, changed } with values
28
+ * { url, added, removed, more? } without: the lines that are new and gone
22
29
  * event "down": { url, error, looks } after three failed looks in a row
23
30
  * event "up": { url } when it answers again
24
31
  *
25
32
  * The first look is the starting point and reports nothing.
33
+ *
34
+ * A rendered look runs in a helper program (mcp/render.ts, started by `rendered` in watching.ts):
35
+ * the page's own scripts run in that browser, so everything it connects to goes through the
36
+ * guard of the browser server, which keeps it off the local network. A local address is opened
37
+ * only where it says by itself that it is local, and then that host and port and no other.
26
38
  */
27
- const MIN = 30_000;
39
+ const MIN = "30s";
40
+ /** With `render` a browser is started for every look. */
41
+ const RENDER_MIN = "2m";
28
42
  const DOWN_AFTER = 3;
29
- const MAX_BODY = 1024 * 1024;
30
- const TIMEOUT = 20_000;
31
- /**
32
- * Whether an address says by itself that it is on this computer or the local network. Such an
33
- * address the owner confirmed as what it is. Any other one must stay on the public internet: a
34
- * name that one day points into the local network is not followed there.
35
- */
36
- const isLocal = (url) => {
37
- const host = url.hostname.replace(/^\[|\]$/g, "");
38
- return isLocalName(host) || (net.isIP(host) !== 0 && isPrivateAddress(host));
39
- };
40
- /** Why a look failed, in words: what lies behind "fetch failed". */
41
- const reason = (e, url) => {
42
- const text = errText(e?.cause ?? e);
43
- return /resolves to a private address/.test(text) ? `${url.hostname} leads to an address on your own network. If that is meant, set the source up again with "local": true` : text;
44
- };
45
- /** Headers that carry a secret. Those go into `token`, which is kept out of the configuration. */
46
- const SECRET_HEADERS = ["authorization", "proxy-authorization", "cookie"];
43
+ const NAME = /^[A-Za-z][A-Za-z0-9_-]{0,39}$/;
47
44
  export const type = {
48
45
  summary: "a web address looked at again and again: reports when what it answers changes (the whole page, or one value in it: a price, a version, a state), and when it stops or starts answering. For pages and services that cannot call you",
49
46
  options: Type.Object({
50
47
  url: Type.String({ description: "the address to look at, https://... (http:// for a device on the local network)" }),
51
48
  every: Type.Optional(Type.String({ description: 'how often to look, e.g. "5m", "1h". Default "5m", at least "30s"' })),
52
49
  path: Type.Optional(Type.String({ description: 'for an answer in JSON: watch only this field, e.g. "data.price" or "items.0.state"' })),
53
- match: Type.Optional(Type.String({ description: 'for a page or text: watch only what this regular expression finds (its first group, if it has one), e.g. "Price: ([0-9.,]+)". Without path or match the whole text counts, and a page that shows the time or a counter changes at every look' })),
50
+ match: Type.Optional(Type.String({ description: 'for a page or text: watch only what this regular expression finds (its first group, if it has one), e.g. "Price: ([0-9.,]+)". Without path, values, match or select the whole text counts, and a page that shows the time or a counter changes at every look' })),
51
+ select: Type.Optional(Type.String({ description: 'for a page: watch only the text of the first element this CSS selector matches, e.g. ".product .price" or "#stock > span". Tags, #id, .class, [attribute=value], nested with a space or >' })),
52
+ attr: Type.Optional(Type.String({ description: 'with "select": watch the value of this attribute of the element instead of its text, e.g. "content" for a meta tag or "data-price"' })),
53
+ render: Type.Optional(Type.Boolean({ description: 'true for a page that a script fills in after loading, where the value is not in the page as the server sends it. Every look then opens the page in pine\'s browser, without a window, and takes what a reader sees. "select" can then be any CSS selector, "every" is at least "2m", and "path", "values", "method", "body", "headers" and "token" do not go with it. Slow and heavy: where the page\'s script asks an address of its own (usually JSON), watch that address without "render" instead' })),
54
+ values: Type.Optional(Type.Record(Type.String(), Type.String(), { description: 'for an answer in JSON: watch several fields at once, each under a name of yours, e.g. {"price": "data.price", "stock": "data.stock.count"}. One event says which of them changed' })),
55
+ method: Type.Optional(Type.String({ description: '"POST" for an address that is asked by sending something (GraphQL, a search). Default "GET"' })),
56
+ body: Type.Optional(Type.String({ description: 'with "POST": what to send, as text, e.g. "{\\"query\\": \\"{ product(id: 7) { price } }\\"}". Sent as JSON when it is JSON. Not for secrets' })),
54
57
  headers: Type.Optional(Type.Record(Type.String(), Type.String(), { description: 'headers to send, e.g. {"accept": "application/json"}. Not for secrets' })),
55
58
  token: Type.Optional(Type.String({ description: "an access token or key, if the address wants one. Sent as Authorization: Bearer <token>", secret: true })),
56
59
  tokenHeader: Type.Optional(Type.String({ description: 'send the token as this header instead, as it is, e.g. "x-api-key"' })),
57
60
  local: Type.Optional(Type.Boolean({ description: "true if the address is on your owner's own network under a name that does not say so (nas.home, ha.lan). Not needed for an IP address, localhost or a .local name" })),
58
61
  }, { additionalProperties: false }),
59
- plan(o) {
60
- let url;
61
- try {
62
- url = new URL(o.url);
63
- }
64
- catch {
65
- throw new Error(`"${o.url}" is not an address. It looks like https://example.com/page`);
66
- }
67
- if (!["http:", "https:"].includes(url.protocol))
68
- throw new Error("the address must start with https:// or http://");
69
- if (url.username || url.password)
70
- throw new Error('an address with a user name or password in it is not taken: leave it out of the address and name "token" in "ask"');
71
- const every = o.every ?? "5m";
72
- if (parseDuration(every) < MIN)
73
- throw new Error('"every" must be at least "30s"');
74
- if (o.path && o.match)
62
+ plan(o, setup) {
63
+ const render = o.render === true || o.render === "true";
64
+ if (render && parseDuration(o.every ?? "5m") < parseDuration(RENDER_MIN))
65
+ throw new Error(`"every" must be at least "${RENDER_MIN}" with "render": a browser is started for every look`);
66
+ const { url, every, local } = askable(o, MIN, "5m");
67
+ if (o.path && o.match && !o.values && !o.select)
75
68
  throw new Error('"path" is for an answer in JSON, "match" for a page or text: give one of them');
69
+ const ways = ["path", "values", "match", "select"].filter((k) => o[k] !== undefined && o[k] !== "");
70
+ if (ways.length > 1)
71
+ throw new Error(`${ways.map((w) => `"${w}"`).join(" and ")} are different ways to say what counts in the answer: give one of them`);
76
72
  if (o.match)
77
73
  pattern(o.match);
78
- const secret = Object.keys(o.headers ?? {}).filter((h) => SECRET_HEADERS.includes(h.toLowerCase()));
79
- if (secret.length)
80
- throw new Error(`the header ${secret.join(", ")} carries a secret: leave it out, name "token" in "ask" and give "tokenHeader": "${secret[0].toLowerCase()}"`);
81
- if (o.tokenHeader && !/^[A-Za-z0-9-]+$/.test(o.tokenHeader))
82
- throw new Error('"tokenHeader" is the name of a header, e.g. "x-api-key"');
83
- const what = o.path ? `the field ${o.path} of what it answers` : o.match ? `what /${o.match}/ finds in it` : "what it answers";
74
+ // A rendered page is read by the browser, which takes every selector there is.
75
+ if (o.select && !render)
76
+ selector(o.select);
77
+ if (o.attr !== undefined && !o.select)
78
+ throw new Error('"attr" goes with "select": it names an attribute of the element the selector finds');
79
+ if (o.attr !== undefined && !/^[A-Za-z_:][\w:.-]*$/.test(o.attr))
80
+ throw new Error('"attr" is the name of an attribute, e.g. "content" or "data-price"');
81
+ const names = Object.keys(o.values ?? {});
82
+ if (o.values !== undefined) {
83
+ if (!names.length)
84
+ throw new Error('"values" is empty. It names the fields to watch: {"price": "data.price", "stock": "data.stock"}');
85
+ if (names.length > 20)
86
+ throw new Error('"values" takes 20 fields at most');
87
+ const odd = names.filter((n) => !NAME.test(n) || !String(o.values[n]).trim());
88
+ if (odd.length)
89
+ throw new Error(`"values": ${odd.join(", ")} will not do. Each entry is a short name of yours (letters, digits, - and _) and the field it stands for: {"price": "data.price"}`);
90
+ }
91
+ const method = String(o.method ?? "GET").toUpperCase();
92
+ if (method !== "GET" && method !== "POST")
93
+ throw new Error('"method" is "GET" or "POST". A source only looks: it never changes or deletes anything');
94
+ if (o.body !== undefined && method !== "POST")
95
+ throw new Error('"body" is what a POST sends: give "method": "POST" with it');
96
+ if (o.body !== undefined && String(o.body).length > 20_000)
97
+ throw new Error('"body" is too long: 20000 characters at most');
98
+ const allowed = render ? renderable(o, url, setup?.controlPort) : undefined;
99
+ const what = o.path ? `the field ${o.path} of what it answers` : names.length ? `one of the fields ${names.map((n) => `${n} (${o.values[n]})`).join(", ")} of what it answers` : o.match ? `what /${o.match}/ finds in it` : o.select ? `${o.attr ? `the attribute ${o.attr} of ` : ""}the element ${o.select} of the page` : render ? "what it shows" : "what it answers";
100
+ const where = `${clip(url.href, 160)}${local ? " (an address on your own network or this computer)" : ""}`;
101
+ const { method: _m, render: _r, ...rest } = o;
102
+ const does = render
103
+ ? [`open ${where} in pine's browser every ${every}, without a window, and report when ${what} changes, and when the page stops or starts opening`, `the page's own scripts run in that browser, as in yours. From there the page reaches the internet${allowed ? ` and ${allowed} on your network, and nothing else on your network` : " and nothing on your own network"}`]
104
+ : [`${method === "POST" ? `send a POST request${o.body ? ` (${clip(String(o.body), 200)})` : ""} to` : "ask"} ${where} every ${every} and report when ${what} changes, and when it stops or starts answering`, ...(o.token ? [`with the token given, sent only to ${url.host}`] : [])];
84
105
  return {
85
- options: { ...o, url: url.href },
86
- does: [`ask ${clip(url.href, 160)}${o.local || isLocal(url) ? " (an address on your own network or this computer)" : ""} every ${every} and report when ${what} changes, and when it stops or starts answering`, ...(o.token ? [`with the token given, sent only to ${url.host}`] : [])],
106
+ options: { ...rest, url: url.href, ...(method === "POST" ? { method } : {}), ...(render ? { render: true } : {}) },
107
+ does,
87
108
  opens: true,
88
109
  tell: "Its first look is the starting point and reports nothing; from then on only a change does. The sources tool shows what it found at the last look.",
89
110
  };
90
111
  },
91
112
  };
92
- /** One look: the answer's text, at most 1 MB of it. A redirect is followed only on the same host. */
93
- async function get(start, o, guard, signal) {
94
- const headers = { ...(o.headers ?? {}) };
95
- if (o.token)
96
- headers[o.tokenHeader ?? "authorization"] = o.tokenHeader ? o.token : `Bearer ${o.token}`;
97
- let at = start;
98
- for (let hop = 0; hop < 6; hop++) {
99
- const res = await fetch(at, { headers, redirect: "manual", signal: AbortSignal.any([signal, AbortSignal.timeout(TIMEOUT)]), ...(guard ? { dispatcher: guard } : {}) });
100
- const to = res.status >= 300 && res.status < 400 ? res.headers.get("location") : null;
101
- if (to) {
102
- await res.body?.cancel().catch(() => { });
103
- const next = new URL(to, at);
104
- // The owner confirmed one host. Another one is another decision, and must not get the token.
105
- const upgrade = at.protocol === "http:" && next.protocol === "https:";
106
- if (next.hostname !== start.hostname || (next.protocol !== at.protocol && !upgrade) || (next.port !== at.port && !upgrade))
107
- throw new Error(`it sends on to ${next.host}, which is not the address that was confirmed. If that is the right one, set the source up with it`);
108
- at = next;
109
- continue;
110
- }
111
- if (!res.ok) {
112
- await res.body?.cancel().catch(() => { });
113
- throw new Error(`it answered ${res.status}`);
114
- }
115
- const chunks = [];
116
- let size = 0;
117
- const reader = res.body?.getReader();
118
- while (reader) {
119
- const { done, value } = await reader.read();
120
- if (done)
121
- break;
122
- chunks.push(Buffer.from(value.subarray(0, MAX_BODY - size)));
123
- size += value.length;
124
- if (size >= MAX_BODY) {
125
- await reader.cancel().catch(() => { });
126
- break;
127
- }
128
- }
129
- return { type: res.headers.get("content-type") ?? "", body: Buffer.concat(chunks).toString("utf8") };
130
- }
131
- throw new Error("it sends on and on (more than five redirects)");
113
+ /**
114
+ * What a source with `render` cannot have, said so that it can be put right, when it is set up and
115
+ * again when it starts. Returns the one local "host:port" the page's browser may reach, if the
116
+ * address is a local one. `control` is the port of pine's own control API, which no page is opened at.
117
+ */
118
+ function renderable(o, url, control) {
119
+ const json = ["path", "values"].filter((k) => o[k] !== undefined && o[k] !== "");
120
+ if (json.length)
121
+ throw new Error(`"${json[0]}" is for an answer in JSON, and "render" is for a page as a browser shows it: leave "render" out to watch JSON`);
122
+ const sent = ["method", "body", "headers", "token", "tokenHeader"].filter((k) => o[k] !== undefined && o[k] !== "" && !(k === "method" && String(o[k]).toUpperCase() === "GET") && !(k === "headers" && !Object.keys(o[k] ?? {}).length));
123
+ if (sent.length)
124
+ throw new Error(`${sent.map((k) => `"${k}"`).join(" and ")} ${sent.length === 1 ? "does" : "do"} not go with "render": a rendered page is asked the way a reader's browser asks it, and a token or a header would go to every address the page loads from. Leave ${sent.length === 1 ? "it" : "them"} out, or leave "render" out and watch the address that needs ${sent.length === 1 ? "it" : "them"}`);
125
+ if (o.select !== undefined && (!String(o.select).trim() || String(o.select).length > 300))
126
+ throw new Error('"select" is a CSS selector of at most 300 characters, e.g. ".product .price"');
127
+ const target = localTarget(url.href);
128
+ // A name that has to be looked up can point somewhere else tomorrow, and a page's scripts would be there then.
129
+ if (!target && o.local === true)
130
+ throw new Error(`"render" does not go with "local": true. A browser is let onto your network only at an address that says by itself that it is local: localhost, a .local name or an IP address. Give ${url.hostname} by its IP address, or leave "render" out`);
131
+ if (target && control !== undefined && target.port === control)
132
+ throw new Error(`${target.key} is pine's own control port. No page is opened there`);
133
+ return target?.key;
132
134
  }
133
135
  export default function http(source) {
134
136
  const o = source.options;
@@ -137,22 +139,44 @@ export default function http(source) {
137
139
  const url = new URL(o.url);
138
140
  if (!["http:", "https:"].includes(url.protocol))
139
141
  throw new Error("the address must start with https:// or http://");
142
+ const render = o.render === true;
140
143
  const re = o.match ? pattern(String(o.match)) : undefined;
144
+ const sel = o.select && !render ? selector(String(o.select)) : undefined;
145
+ const fields = o.values && typeof o.values === "object" ? Object.entries(o.values).map(([k, v]) => [k, String(v)]) : [];
141
146
  source.secret(o.token);
142
- const guard = o.local === true || isLocal(url) ? undefined : publicOnlyAgent(TIMEOUT);
147
+ // The one local address the page's browser may reach, written where the browser's guard reads it and no agent writes.
148
+ let localFile;
149
+ if (render) {
150
+ const allowed = renderable(o, url, source.controlPort);
151
+ const file = path.join(source.dataDir, "local.json");
152
+ // (Written anew at every start: the address may be another one than last time, or not a local one any more.)
153
+ fs.rmSync(file, { force: true });
154
+ if (allowed)
155
+ fs.writeFileSync((localFile = file), JSON.stringify([allowed]), { mode: 0o600 });
156
+ }
157
+ const guard = render ? undefined : guardFor(url, o.local);
143
158
  if (guard)
144
159
  source.onStop(() => guard.close());
160
+ // What an answer is read for: an address that says "nothing new" is believed only for the same reading.
161
+ const about = { url: url.href, path: o.path, values: o.values, match: o.match, select: o.select, attr: o.attr };
145
162
  // How many looks in a row failed. Whether "down" was said is kept, so that "up" follows it also across a restart.
146
163
  let failed = 0;
164
+ // A look with the browser that is under way. The source has stopped only when its browser is gone.
165
+ let looking;
166
+ source.onStop(async () => void (await looking?.catch(() => { })));
147
167
  source.poll(o.every ?? "5m", async (signal) => {
148
168
  let answer;
169
+ let page;
149
170
  try {
150
- answer = await get(url, o, guard, signal);
171
+ if (render)
172
+ page = await (looking = rendered({ url: url.href, select: o.select ? String(o.select) : undefined, attr: o.attr ? String(o.attr) : undefined, localFile }, source.dataDir, signal));
173
+ else
174
+ answer = await ask(url, o, guard, signal, sinceOf(source, about));
151
175
  }
152
176
  catch (e) {
153
177
  if (signal.aborted)
154
178
  return;
155
- const why = reason(e, url);
179
+ const why = render ? errText(e) : whyFailed(e, url);
156
180
  failed++;
157
181
  if (failed === DOWN_AFTER && !source.kv.get("down")) {
158
182
  source.kv.set("down", true);
@@ -165,32 +189,59 @@ export default function http(source) {
165
189
  source.kv.set("down", false);
166
190
  source.emit("up", { url: o.url });
167
191
  }
168
- const html = /html/i.test(answer.type) || /^\s*<(!doctype html|html)\b/i.test(answer.body);
192
+ // Nothing new since the last answer, says the address itself: what was found then still stands.
193
+ if (answer?.unchanged)
194
+ return;
195
+ const body = answer?.body ?? "";
196
+ const html = Boolean(answer) && (/html/i.test(answer.type) || /^\s*<(!doctype html|html)\b/i.test(body));
169
197
  let json;
170
- if (!html) {
198
+ if (answer && !html) {
171
199
  try {
172
- json = JSON.parse(answer.body);
200
+ json = JSON.parse(body);
173
201
  }
174
202
  catch {
175
203
  // plain text
176
204
  }
177
205
  }
178
- if (o.path || re) {
206
+ if (fields.length) {
207
+ const values = Object.fromEntries(fields.map(([name, path]) => [name, json === undefined ? null : (getPath(json, path) ?? null)]));
208
+ const missing = fields.filter(([name]) => values[name] === null).length;
209
+ source.status(missing === fields.length ? "found none of the fields" : `now: ${fields.slice(0, 4).map(([name]) => `${name} ${harmless(values[name])}`).join(", ")}${fields.length > 4 ? ` and ${fields.length - 4} more` : ""}`);
210
+ const was = source.changed(values);
211
+ keepSince(source, about, answer);
212
+ if (!was)
213
+ return;
214
+ const previous = was.previous && typeof was.previous === "object" ? was.previous : {};
215
+ const changed = fields.map(([name]) => name).filter((name) => JSON.stringify(values[name]) !== JSON.stringify(previous[name] ?? null));
216
+ if (changed.length)
217
+ source.emit("changed", { url: o.url, values: Object.fromEntries(fields.map(([name]) => [name, shown(values[name])])), previous: Object.fromEntries(fields.map(([name]) => [name, shown(previous[name] ?? null)])), changed });
218
+ return;
219
+ }
220
+ if (o.path || re || o.select) {
179
221
  let value = null;
180
- if (o.path)
222
+ // A rendered page: the browser found the element, and a pattern looks in what a reader sees.
223
+ if (page)
224
+ value = o.select ? (typeof page.value === "string" ? clip(page.value.trim()) : null) : found(re, page.text ?? "");
225
+ else if (o.path)
181
226
  value = json === undefined ? null : (getPath(json, String(o.path)) ?? null);
227
+ else if (sel)
228
+ value = selected(body, sel, o.attr ? String(o.attr) : undefined);
182
229
  // In what a reader sees first; a pattern that is about the markup finds it in the source.
183
230
  else
184
- value = (html ? found(re, pageText(answer.body)) : null) ?? found(re, answer.body);
185
- source.status(value === null ? `found nothing ${o.path ? `at ${o.path}` : "with the pattern"}` : `now: ${harmless(value)}`);
231
+ value = (html ? found(re, pageText(body)) : null) ?? found(re, body);
232
+ source.status(value === null ? `found nothing ${o.path ? `at ${o.path}` : o.select ? "with the selector" : "with the pattern"}` : `now: ${harmless(value)}`);
186
233
  const was = source.changed(value);
234
+ if (answer)
235
+ keepSince(source, about, answer);
187
236
  if (was)
188
- source.emit("changed", { url: o.url, value: shown(value), previous: shown(was.previous) });
237
+ source.emit("changed", { url: o.url, value: shown(value), previous: shown(was.previous), ...numbers(value, was.previous) });
189
238
  return;
190
239
  }
191
- const text = (html ? pageText(answer.body) : json !== undefined ? JSON.stringify(json, null, 1) : answer.body).slice(0, 200_000);
240
+ const text = (page ? (page.text ?? "") : html ? pageText(body) : json !== undefined ? JSON.stringify(json, null, 1) : body).slice(0, 200_000);
192
241
  source.status(`now: ${text.split("\n").length} lines`);
193
242
  const was = source.changed(text);
243
+ if (answer)
244
+ keepSince(source, about, answer);
194
245
  if (!was)
195
246
  return;
196
247
  const diff = lineDiff(String(was.previous), text);
@@ -0,0 +1,51 @@
1
+ import type { SourceType } from "../source-types.ts";
2
+ import type { SourceApi } from "../sources.ts";
3
+ /**
4
+ * A mailbox: reports every new mail, the moment the server says there is one (IMAP IDLE).
5
+ *
6
+ * options: { host, user, password, port?, security?, mailbox?, from?, subject?, maxMb?, keepDays?,
7
+ * smtpHost?, smtpPort?, sendTo?, sender?, insecureTls?, local? }
8
+ * security "tls" (default, port 993), "starttls" (port 143), "none" only for a server on this machine or network
9
+ * mailbox which folder to watch ("INBOX")
10
+ * from only mail that says it is from one of these: an address, or "@domain"
11
+ * subject only mail whose subject this regular expression finds
12
+ * maxMb attachments are kept up to this size (10)
13
+ * sendTo addresses the agent may write to, through smtpHost. Without it nothing can be sent
14
+ * signIn "microsoft": no password. The owner signs in at Microsoft in a browser, once, with a
15
+ * code pine shows (the device code flow), and pine logs in with what it is given
16
+ * (XOAUTH2). For Outlook.com, Hotmail and Microsoft 365, which take no password over IMAP.
17
+ * Needs `clientId`: an app the owner registered at Microsoft; `tenant` says for which accounts.
18
+ *
19
+ * event "mail": { from, fromName?, to, subject, date?, text, attachments: [{ file, path?, type, size }], images?, cut? }
20
+ * It can send, to the addresses on `sendTo` only: { to, subject, text }
21
+ *
22
+ * It only reads: the mailbox is opened read-only (EXAMINE) and messages are fetched with PEEK, so
23
+ * nothing is marked as read, moved or deleted. Mail that is there when it first connects is the
24
+ * starting point and is not reported. Who a mail says it is from is what the sender wrote, and
25
+ * anyone can write any address: a mail is outside content, whoever it claims to come from.
26
+ */
27
+ /** How long things take. A test makes them short. (`second`: what a second of waiting between two questions to Microsoft lasts.) */
28
+ export declare const pace: {
29
+ retry: number;
30
+ retryMax: number;
31
+ idle: number;
32
+ noop: number;
33
+ timeout: number;
34
+ second: number;
35
+ };
36
+ /**
37
+ * Where Microsoft signs people in. A test puts its stand-in here. It is not an option of the
38
+ * source: whoever names this address is handed the sign-in, so no agent and no configuration does.
39
+ */
40
+ export declare const oauth: {
41
+ authority: string;
42
+ };
43
+ export declare const type: SourceType;
44
+ /** A message as it goes over the wire: headers of pine's making, the text as base64. */
45
+ export declare function composeMail(m: {
46
+ from: string;
47
+ to: string;
48
+ subject: string;
49
+ text: string;
50
+ }): string;
51
+ export default function mail(source: SourceApi): Promise<void>;