@pmndre/pine 0.22.1 → 0.33.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/changes.json +265 -0
- package/docs/SKILL.md +31 -7
- package/docs/references/agents.md +245 -21
- package/docs/references/coding.md +1 -1
- package/docs/references/computer.md +1 -1
- package/docs/references/configuration.md +14 -5
- package/docs/references/mcp.md +9 -0
- package/docs/references/operations.md +20 -9
- package/docs/references/overview.md +3 -1
- package/docs/references/security.md +83 -24
- package/docs/references/setup.md +55 -9
- package/docs/references/source-types.md +1418 -0
- package/docs/references/sources.md +333 -19
- package/docs/references/stack.md +1 -0
- package/docs/references/tui.md +86 -12
- package/examples/home/AGENTS.md +4 -0
- package/examples/home/TOOLS.md +5 -0
- package/examples/home/agents/main/workspace/AGENTS.md +1 -1
- package/examples/home/agents/main/workspace/TOOLS.md +5 -0
- package/mcp/browser.d.ts +6 -0
- package/mcp/browser.js +19 -0
- package/mcp/render.d.ts +1 -0
- package/mcp/render.js +237 -0
- package/package.json +3 -1
- package/src/actions.d.ts +6 -1
- package/src/actions.js +17 -4
- package/src/admin.d.ts +4 -0
- package/src/admin.js +25 -6
- package/src/agents.d.ts +167 -2
- package/src/agents.js +912 -45
- package/src/catalogue.d.ts +60 -0
- package/src/catalogue.js +791 -0
- package/src/cli.js +10 -7
- package/src/config.d.ts +53 -4
- package/src/config.js +30 -5
- package/src/goals.d.ts +1 -1
- package/src/goals.js +14 -1
- package/src/manual.js +24 -2
- package/src/mime.d.ts +78 -0
- package/src/mime.js +342 -0
- package/src/net.d.ts +11 -0
- package/src/net.js +29 -2
- package/src/pine.js +10 -1
- package/src/router.js +3 -3
- package/src/sandbox-linux.d.ts +3 -1
- package/src/sandbox-linux.js +5 -0
- package/src/security.d.ts +22 -6
- package/src/security.js +44 -13
- package/src/slots.d.ts +106 -0
- package/src/slots.js +250 -0
- package/src/source-types.d.ts +20 -0
- package/src/sources/calendar.d.ts +24 -0
- package/src/sources/calendar.js +265 -0
- package/src/sources/command.js +109 -4
- package/src/sources/discord.d.ts +4 -0
- package/src/sources/discord.js +298 -0
- package/src/sources/feed.d.ts +19 -0
- package/src/sources/feed.js +257 -0
- package/src/sources/http.js +154 -103
- package/src/sources/mail.d.ts +51 -0
- package/src/sources/mail.js +1013 -0
- package/src/sources/matrix.d.ts +4 -0
- package/src/sources/matrix.js +232 -0
- package/src/sources/modbus.d.ts +15 -0
- package/src/sources/modbus.js +240 -0
- package/src/sources/onvif.d.ts +35 -0
- package/src/sources/onvif.js +326 -0
- package/src/sources/presence.d.ts +80 -0
- package/src/sources/presence.js +480 -0
- package/src/sources/serial.d.ts +21 -0
- package/src/sources/serial.js +220 -0
- package/src/sources/slack.d.ts +16 -0
- package/src/sources/slack.js +1062 -0
- package/src/sources/smtp.d.ts +4 -0
- package/src/sources/smtp.js +360 -0
- package/src/sources/stream.d.ts +68 -0
- package/src/sources/stream.js +534 -0
- package/src/sources/syslog.d.ts +32 -0
- package/src/sources/syslog.js +243 -0
- package/src/sources/system.d.ts +14 -0
- package/src/sources/system.js +236 -0
- package/src/sources/system.swift +102 -0
- package/src/sources/telegram.js +34 -1
- package/src/sources.d.ts +41 -1
- package/src/sources.js +147 -24
- package/src/tools.d.ts +1 -1
- package/src/tools.js +304 -23
- package/src/tui/app.d.ts +35 -2
- package/src/tui/app.js +306 -43
- package/src/tui/levels.d.ts +68 -0
- package/src/tui/levels.js +273 -0
- package/src/tui/picture.d.ts +17 -1
- package/src/tui/picture.js +72 -5
- package/src/types.d.ts +3 -0
- package/src/update.d.ts +9 -1
- package/src/update.js +75 -3
- package/src/watching.d.ts +156 -0
- package/src/watching.js +609 -4
package/src/sources/http.js
CHANGED
|
@@ -1,134 +1,136 @@
|
|
|
1
|
-
import
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
2
3
|
import { Type } from "typebox";
|
|
3
|
-
import {
|
|
4
|
-
import { isLocalName, isPrivateAddress, publicOnlyAgent } from "../net.js";
|
|
4
|
+
import { localTarget } from "../net.js";
|
|
5
5
|
import { errText, getPath, parseDuration } from "../util.js";
|
|
6
|
-
import { clip, found, harmless, lineDiff, pageText, pattern, shown } from "../watching.js";
|
|
6
|
+
import { ask, askable, clip, found, guardFor, harmless, keepSince, lineDiff, numbers, pageText, pattern, rendered, selected, selector, shown, sinceOf, whyFailed } from "../watching.js";
|
|
7
7
|
/**
|
|
8
8
|
* A web address, looked at again and again: reports when what it answers changes, and when it
|
|
9
9
|
* stops or starts answering. For pages and services that cannot call you: a price, a version, a
|
|
10
10
|
* status page, a device with a state page on your network.
|
|
11
11
|
*
|
|
12
|
-
* options: { url, every?, path?, match?, headers?, token?, tokenHeader?, local? }
|
|
12
|
+
* options: { url, every?, path?, values?, match?, select?, attr?, render?, method?, body?, headers?, token?, tokenHeader?, local? }
|
|
13
13
|
* every how often to look ("5m" by default)
|
|
14
14
|
* path for a JSON answer: watch only this field ("data.price")
|
|
15
|
+
* values for a JSON answer: watch several fields, each under a name ({ "price": "data.price" })
|
|
15
16
|
* match for a page or text: watch only what this regular expression finds
|
|
17
|
+
* select for a page: watch only the text of the first element this CSS selector matches
|
|
18
|
+
* attr with select: the value of this attribute of the element instead of its text
|
|
19
|
+
* render for a page that a script fills in after loading: every look opens the page in pine's browser,
|
|
20
|
+
* without a window, and takes what a reader sees then. `select` is the browser's own CSS then
|
|
21
|
+
* method, body "POST" and what to send, for an address that is asked that way (GraphQL)
|
|
16
22
|
* headers headers to send
|
|
17
23
|
* token sent as "Authorization: Bearer <token>", or as it is in the header `tokenHeader`
|
|
18
24
|
* local the address is on the owner's own network, under a name that does not say so
|
|
19
25
|
*
|
|
20
|
-
* event "changed": { url, value, previous }
|
|
21
|
-
* { url,
|
|
26
|
+
* event "changed": { url, value, previous, number?, previousNumber? } with path, match or select
|
|
27
|
+
* { url, values, previous, changed } with values
|
|
28
|
+
* { url, added, removed, more? } without: the lines that are new and gone
|
|
22
29
|
* event "down": { url, error, looks } after three failed looks in a row
|
|
23
30
|
* event "up": { url } when it answers again
|
|
24
31
|
*
|
|
25
32
|
* The first look is the starting point and reports nothing.
|
|
33
|
+
*
|
|
34
|
+
* A rendered look runs in a helper program (mcp/render.ts, started by `rendered` in watching.ts):
|
|
35
|
+
* the page's own scripts run in that browser, so everything it connects to goes through the
|
|
36
|
+
* guard of the browser server, which keeps it off the local network. A local address is opened
|
|
37
|
+
* only where it says by itself that it is local, and then that host and port and no other.
|
|
26
38
|
*/
|
|
27
|
-
const MIN =
|
|
39
|
+
const MIN = "30s";
|
|
40
|
+
/** With `render` a browser is started for every look. */
|
|
41
|
+
const RENDER_MIN = "2m";
|
|
28
42
|
const DOWN_AFTER = 3;
|
|
29
|
-
const
|
|
30
|
-
const TIMEOUT = 20_000;
|
|
31
|
-
/**
|
|
32
|
-
* Whether an address says by itself that it is on this computer or the local network. Such an
|
|
33
|
-
* address the owner confirmed as what it is. Any other one must stay on the public internet: a
|
|
34
|
-
* name that one day points into the local network is not followed there.
|
|
35
|
-
*/
|
|
36
|
-
const isLocal = (url) => {
|
|
37
|
-
const host = url.hostname.replace(/^\[|\]$/g, "");
|
|
38
|
-
return isLocalName(host) || (net.isIP(host) !== 0 && isPrivateAddress(host));
|
|
39
|
-
};
|
|
40
|
-
/** Why a look failed, in words: what lies behind "fetch failed". */
|
|
41
|
-
const reason = (e, url) => {
|
|
42
|
-
const text = errText(e?.cause ?? e);
|
|
43
|
-
return /resolves to a private address/.test(text) ? `${url.hostname} leads to an address on your own network. If that is meant, set the source up again with "local": true` : text;
|
|
44
|
-
};
|
|
45
|
-
/** Headers that carry a secret. Those go into `token`, which is kept out of the configuration. */
|
|
46
|
-
const SECRET_HEADERS = ["authorization", "proxy-authorization", "cookie"];
|
|
43
|
+
const NAME = /^[A-Za-z][A-Za-z0-9_-]{0,39}$/;
|
|
47
44
|
export const type = {
|
|
48
45
|
summary: "a web address looked at again and again: reports when what it answers changes (the whole page, or one value in it: a price, a version, a state), and when it stops or starts answering. For pages and services that cannot call you",
|
|
49
46
|
options: Type.Object({
|
|
50
47
|
url: Type.String({ description: "the address to look at, https://... (http:// for a device on the local network)" }),
|
|
51
48
|
every: Type.Optional(Type.String({ description: 'how often to look, e.g. "5m", "1h". Default "5m", at least "30s"' })),
|
|
52
49
|
path: Type.Optional(Type.String({ description: 'for an answer in JSON: watch only this field, e.g. "data.price" or "items.0.state"' })),
|
|
53
|
-
match: Type.Optional(Type.String({ description: 'for a page or text: watch only what this regular expression finds (its first group, if it has one), e.g. "Price: ([0-9.,]+)". Without path
|
|
50
|
+
match: Type.Optional(Type.String({ description: 'for a page or text: watch only what this regular expression finds (its first group, if it has one), e.g. "Price: ([0-9.,]+)". Without path, values, match or select the whole text counts, and a page that shows the time or a counter changes at every look' })),
|
|
51
|
+
select: Type.Optional(Type.String({ description: 'for a page: watch only the text of the first element this CSS selector matches, e.g. ".product .price" or "#stock > span". Tags, #id, .class, [attribute=value], nested with a space or >' })),
|
|
52
|
+
attr: Type.Optional(Type.String({ description: 'with "select": watch the value of this attribute of the element instead of its text, e.g. "content" for a meta tag or "data-price"' })),
|
|
53
|
+
render: Type.Optional(Type.Boolean({ description: 'true for a page that a script fills in after loading, where the value is not in the page as the server sends it. Every look then opens the page in pine\'s browser, without a window, and takes what a reader sees. "select" can then be any CSS selector, "every" is at least "2m", and "path", "values", "method", "body", "headers" and "token" do not go with it. Slow and heavy: where the page\'s script asks an address of its own (usually JSON), watch that address without "render" instead' })),
|
|
54
|
+
values: Type.Optional(Type.Record(Type.String(), Type.String(), { description: 'for an answer in JSON: watch several fields at once, each under a name of yours, e.g. {"price": "data.price", "stock": "data.stock.count"}. One event says which of them changed' })),
|
|
55
|
+
method: Type.Optional(Type.String({ description: '"POST" for an address that is asked by sending something (GraphQL, a search). Default "GET"' })),
|
|
56
|
+
body: Type.Optional(Type.String({ description: 'with "POST": what to send, as text, e.g. "{\\"query\\": \\"{ product(id: 7) { price } }\\"}". Sent as JSON when it is JSON. Not for secrets' })),
|
|
54
57
|
headers: Type.Optional(Type.Record(Type.String(), Type.String(), { description: 'headers to send, e.g. {"accept": "application/json"}. Not for secrets' })),
|
|
55
58
|
token: Type.Optional(Type.String({ description: "an access token or key, if the address wants one. Sent as Authorization: Bearer <token>", secret: true })),
|
|
56
59
|
tokenHeader: Type.Optional(Type.String({ description: 'send the token as this header instead, as it is, e.g. "x-api-key"' })),
|
|
57
60
|
local: Type.Optional(Type.Boolean({ description: "true if the address is on your owner's own network under a name that does not say so (nas.home, ha.lan). Not needed for an IP address, localhost or a .local name" })),
|
|
58
61
|
}, { additionalProperties: false }),
|
|
59
|
-
plan(o) {
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
throw new Error(`"${o.url}" is not an address. It looks like https://example.com/page`);
|
|
66
|
-
}
|
|
67
|
-
if (!["http:", "https:"].includes(url.protocol))
|
|
68
|
-
throw new Error("the address must start with https:// or http://");
|
|
69
|
-
if (url.username || url.password)
|
|
70
|
-
throw new Error('an address with a user name or password in it is not taken: leave it out of the address and name "token" in "ask"');
|
|
71
|
-
const every = o.every ?? "5m";
|
|
72
|
-
if (parseDuration(every) < MIN)
|
|
73
|
-
throw new Error('"every" must be at least "30s"');
|
|
74
|
-
if (o.path && o.match)
|
|
62
|
+
plan(o, setup) {
|
|
63
|
+
const render = o.render === true || o.render === "true";
|
|
64
|
+
if (render && parseDuration(o.every ?? "5m") < parseDuration(RENDER_MIN))
|
|
65
|
+
throw new Error(`"every" must be at least "${RENDER_MIN}" with "render": a browser is started for every look`);
|
|
66
|
+
const { url, every, local } = askable(o, MIN, "5m");
|
|
67
|
+
if (o.path && o.match && !o.values && !o.select)
|
|
75
68
|
throw new Error('"path" is for an answer in JSON, "match" for a page or text: give one of them');
|
|
69
|
+
const ways = ["path", "values", "match", "select"].filter((k) => o[k] !== undefined && o[k] !== "");
|
|
70
|
+
if (ways.length > 1)
|
|
71
|
+
throw new Error(`${ways.map((w) => `"${w}"`).join(" and ")} are different ways to say what counts in the answer: give one of them`);
|
|
76
72
|
if (o.match)
|
|
77
73
|
pattern(o.match);
|
|
78
|
-
|
|
79
|
-
if (
|
|
80
|
-
|
|
81
|
-
if (o.
|
|
82
|
-
throw new Error('"
|
|
83
|
-
|
|
74
|
+
// A rendered page is read by the browser, which takes every selector there is.
|
|
75
|
+
if (o.select && !render)
|
|
76
|
+
selector(o.select);
|
|
77
|
+
if (o.attr !== undefined && !o.select)
|
|
78
|
+
throw new Error('"attr" goes with "select": it names an attribute of the element the selector finds');
|
|
79
|
+
if (o.attr !== undefined && !/^[A-Za-z_:][\w:.-]*$/.test(o.attr))
|
|
80
|
+
throw new Error('"attr" is the name of an attribute, e.g. "content" or "data-price"');
|
|
81
|
+
const names = Object.keys(o.values ?? {});
|
|
82
|
+
if (o.values !== undefined) {
|
|
83
|
+
if (!names.length)
|
|
84
|
+
throw new Error('"values" is empty. It names the fields to watch: {"price": "data.price", "stock": "data.stock"}');
|
|
85
|
+
if (names.length > 20)
|
|
86
|
+
throw new Error('"values" takes 20 fields at most');
|
|
87
|
+
const odd = names.filter((n) => !NAME.test(n) || !String(o.values[n]).trim());
|
|
88
|
+
if (odd.length)
|
|
89
|
+
throw new Error(`"values": ${odd.join(", ")} will not do. Each entry is a short name of yours (letters, digits, - and _) and the field it stands for: {"price": "data.price"}`);
|
|
90
|
+
}
|
|
91
|
+
const method = String(o.method ?? "GET").toUpperCase();
|
|
92
|
+
if (method !== "GET" && method !== "POST")
|
|
93
|
+
throw new Error('"method" is "GET" or "POST". A source only looks: it never changes or deletes anything');
|
|
94
|
+
if (o.body !== undefined && method !== "POST")
|
|
95
|
+
throw new Error('"body" is what a POST sends: give "method": "POST" with it');
|
|
96
|
+
if (o.body !== undefined && String(o.body).length > 20_000)
|
|
97
|
+
throw new Error('"body" is too long: 20000 characters at most');
|
|
98
|
+
const allowed = render ? renderable(o, url, setup?.controlPort) : undefined;
|
|
99
|
+
const what = o.path ? `the field ${o.path} of what it answers` : names.length ? `one of the fields ${names.map((n) => `${n} (${o.values[n]})`).join(", ")} of what it answers` : o.match ? `what /${o.match}/ finds in it` : o.select ? `${o.attr ? `the attribute ${o.attr} of ` : ""}the element ${o.select} of the page` : render ? "what it shows" : "what it answers";
|
|
100
|
+
const where = `${clip(url.href, 160)}${local ? " (an address on your own network or this computer)" : ""}`;
|
|
101
|
+
const { method: _m, render: _r, ...rest } = o;
|
|
102
|
+
const does = render
|
|
103
|
+
? [`open ${where} in pine's browser every ${every}, without a window, and report when ${what} changes, and when the page stops or starts opening`, `the page's own scripts run in that browser, as in yours. From there the page reaches the internet${allowed ? ` and ${allowed} on your network, and nothing else on your network` : " and nothing on your own network"}`]
|
|
104
|
+
: [`${method === "POST" ? `send a POST request${o.body ? ` (${clip(String(o.body), 200)})` : ""} to` : "ask"} ${where} every ${every} and report when ${what} changes, and when it stops or starts answering`, ...(o.token ? [`with the token given, sent only to ${url.host}`] : [])];
|
|
84
105
|
return {
|
|
85
|
-
options: { ...
|
|
86
|
-
does
|
|
106
|
+
options: { ...rest, url: url.href, ...(method === "POST" ? { method } : {}), ...(render ? { render: true } : {}) },
|
|
107
|
+
does,
|
|
87
108
|
opens: true,
|
|
88
109
|
tell: "Its first look is the starting point and reports nothing; from then on only a change does. The sources tool shows what it found at the last look.",
|
|
89
110
|
};
|
|
90
111
|
},
|
|
91
112
|
};
|
|
92
|
-
/**
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
throw new Error(`it answered ${res.status}`);
|
|
114
|
-
}
|
|
115
|
-
const chunks = [];
|
|
116
|
-
let size = 0;
|
|
117
|
-
const reader = res.body?.getReader();
|
|
118
|
-
while (reader) {
|
|
119
|
-
const { done, value } = await reader.read();
|
|
120
|
-
if (done)
|
|
121
|
-
break;
|
|
122
|
-
chunks.push(Buffer.from(value.subarray(0, MAX_BODY - size)));
|
|
123
|
-
size += value.length;
|
|
124
|
-
if (size >= MAX_BODY) {
|
|
125
|
-
await reader.cancel().catch(() => { });
|
|
126
|
-
break;
|
|
127
|
-
}
|
|
128
|
-
}
|
|
129
|
-
return { type: res.headers.get("content-type") ?? "", body: Buffer.concat(chunks).toString("utf8") };
|
|
130
|
-
}
|
|
131
|
-
throw new Error("it sends on and on (more than five redirects)");
|
|
113
|
+
/**
|
|
114
|
+
* What a source with `render` cannot have, said so that it can be put right, when it is set up and
|
|
115
|
+
* again when it starts. Returns the one local "host:port" the page's browser may reach, if the
|
|
116
|
+
* address is a local one. `control` is the port of pine's own control API, which no page is opened at.
|
|
117
|
+
*/
|
|
118
|
+
function renderable(o, url, control) {
|
|
119
|
+
const json = ["path", "values"].filter((k) => o[k] !== undefined && o[k] !== "");
|
|
120
|
+
if (json.length)
|
|
121
|
+
throw new Error(`"${json[0]}" is for an answer in JSON, and "render" is for a page as a browser shows it: leave "render" out to watch JSON`);
|
|
122
|
+
const sent = ["method", "body", "headers", "token", "tokenHeader"].filter((k) => o[k] !== undefined && o[k] !== "" && !(k === "method" && String(o[k]).toUpperCase() === "GET") && !(k === "headers" && !Object.keys(o[k] ?? {}).length));
|
|
123
|
+
if (sent.length)
|
|
124
|
+
throw new Error(`${sent.map((k) => `"${k}"`).join(" and ")} ${sent.length === 1 ? "does" : "do"} not go with "render": a rendered page is asked the way a reader's browser asks it, and a token or a header would go to every address the page loads from. Leave ${sent.length === 1 ? "it" : "them"} out, or leave "render" out and watch the address that needs ${sent.length === 1 ? "it" : "them"}`);
|
|
125
|
+
if (o.select !== undefined && (!String(o.select).trim() || String(o.select).length > 300))
|
|
126
|
+
throw new Error('"select" is a CSS selector of at most 300 characters, e.g. ".product .price"');
|
|
127
|
+
const target = localTarget(url.href);
|
|
128
|
+
// A name that has to be looked up can point somewhere else tomorrow, and a page's scripts would be there then.
|
|
129
|
+
if (!target && o.local === true)
|
|
130
|
+
throw new Error(`"render" does not go with "local": true. A browser is let onto your network only at an address that says by itself that it is local: localhost, a .local name or an IP address. Give ${url.hostname} by its IP address, or leave "render" out`);
|
|
131
|
+
if (target && control !== undefined && target.port === control)
|
|
132
|
+
throw new Error(`${target.key} is pine's own control port. No page is opened there`);
|
|
133
|
+
return target?.key;
|
|
132
134
|
}
|
|
133
135
|
export default function http(source) {
|
|
134
136
|
const o = source.options;
|
|
@@ -137,22 +139,44 @@ export default function http(source) {
|
|
|
137
139
|
const url = new URL(o.url);
|
|
138
140
|
if (!["http:", "https:"].includes(url.protocol))
|
|
139
141
|
throw new Error("the address must start with https:// or http://");
|
|
142
|
+
const render = o.render === true;
|
|
140
143
|
const re = o.match ? pattern(String(o.match)) : undefined;
|
|
144
|
+
const sel = o.select && !render ? selector(String(o.select)) : undefined;
|
|
145
|
+
const fields = o.values && typeof o.values === "object" ? Object.entries(o.values).map(([k, v]) => [k, String(v)]) : [];
|
|
141
146
|
source.secret(o.token);
|
|
142
|
-
|
|
147
|
+
// The one local address the page's browser may reach, written where the browser's guard reads it and no agent writes.
|
|
148
|
+
let localFile;
|
|
149
|
+
if (render) {
|
|
150
|
+
const allowed = renderable(o, url, source.controlPort);
|
|
151
|
+
const file = path.join(source.dataDir, "local.json");
|
|
152
|
+
// (Written anew at every start: the address may be another one than last time, or not a local one any more.)
|
|
153
|
+
fs.rmSync(file, { force: true });
|
|
154
|
+
if (allowed)
|
|
155
|
+
fs.writeFileSync((localFile = file), JSON.stringify([allowed]), { mode: 0o600 });
|
|
156
|
+
}
|
|
157
|
+
const guard = render ? undefined : guardFor(url, o.local);
|
|
143
158
|
if (guard)
|
|
144
159
|
source.onStop(() => guard.close());
|
|
160
|
+
// What an answer is read for: an address that says "nothing new" is believed only for the same reading.
|
|
161
|
+
const about = { url: url.href, path: o.path, values: o.values, match: o.match, select: o.select, attr: o.attr };
|
|
145
162
|
// How many looks in a row failed. Whether "down" was said is kept, so that "up" follows it also across a restart.
|
|
146
163
|
let failed = 0;
|
|
164
|
+
// A look with the browser that is under way. The source has stopped only when its browser is gone.
|
|
165
|
+
let looking;
|
|
166
|
+
source.onStop(async () => void (await looking?.catch(() => { })));
|
|
147
167
|
source.poll(o.every ?? "5m", async (signal) => {
|
|
148
168
|
let answer;
|
|
169
|
+
let page;
|
|
149
170
|
try {
|
|
150
|
-
|
|
171
|
+
if (render)
|
|
172
|
+
page = await (looking = rendered({ url: url.href, select: o.select ? String(o.select) : undefined, attr: o.attr ? String(o.attr) : undefined, localFile }, source.dataDir, signal));
|
|
173
|
+
else
|
|
174
|
+
answer = await ask(url, o, guard, signal, sinceOf(source, about));
|
|
151
175
|
}
|
|
152
176
|
catch (e) {
|
|
153
177
|
if (signal.aborted)
|
|
154
178
|
return;
|
|
155
|
-
const why =
|
|
179
|
+
const why = render ? errText(e) : whyFailed(e, url);
|
|
156
180
|
failed++;
|
|
157
181
|
if (failed === DOWN_AFTER && !source.kv.get("down")) {
|
|
158
182
|
source.kv.set("down", true);
|
|
@@ -165,32 +189,59 @@ export default function http(source) {
|
|
|
165
189
|
source.kv.set("down", false);
|
|
166
190
|
source.emit("up", { url: o.url });
|
|
167
191
|
}
|
|
168
|
-
|
|
192
|
+
// Nothing new since the last answer, says the address itself: what was found then still stands.
|
|
193
|
+
if (answer?.unchanged)
|
|
194
|
+
return;
|
|
195
|
+
const body = answer?.body ?? "";
|
|
196
|
+
const html = Boolean(answer) && (/html/i.test(answer.type) || /^\s*<(!doctype html|html)\b/i.test(body));
|
|
169
197
|
let json;
|
|
170
|
-
if (!html) {
|
|
198
|
+
if (answer && !html) {
|
|
171
199
|
try {
|
|
172
|
-
json = JSON.parse(
|
|
200
|
+
json = JSON.parse(body);
|
|
173
201
|
}
|
|
174
202
|
catch {
|
|
175
203
|
// plain text
|
|
176
204
|
}
|
|
177
205
|
}
|
|
178
|
-
if (
|
|
206
|
+
if (fields.length) {
|
|
207
|
+
const values = Object.fromEntries(fields.map(([name, path]) => [name, json === undefined ? null : (getPath(json, path) ?? null)]));
|
|
208
|
+
const missing = fields.filter(([name]) => values[name] === null).length;
|
|
209
|
+
source.status(missing === fields.length ? "found none of the fields" : `now: ${fields.slice(0, 4).map(([name]) => `${name} ${harmless(values[name])}`).join(", ")}${fields.length > 4 ? ` and ${fields.length - 4} more` : ""}`);
|
|
210
|
+
const was = source.changed(values);
|
|
211
|
+
keepSince(source, about, answer);
|
|
212
|
+
if (!was)
|
|
213
|
+
return;
|
|
214
|
+
const previous = was.previous && typeof was.previous === "object" ? was.previous : {};
|
|
215
|
+
const changed = fields.map(([name]) => name).filter((name) => JSON.stringify(values[name]) !== JSON.stringify(previous[name] ?? null));
|
|
216
|
+
if (changed.length)
|
|
217
|
+
source.emit("changed", { url: o.url, values: Object.fromEntries(fields.map(([name]) => [name, shown(values[name])])), previous: Object.fromEntries(fields.map(([name]) => [name, shown(previous[name] ?? null)])), changed });
|
|
218
|
+
return;
|
|
219
|
+
}
|
|
220
|
+
if (o.path || re || o.select) {
|
|
179
221
|
let value = null;
|
|
180
|
-
|
|
222
|
+
// A rendered page: the browser found the element, and a pattern looks in what a reader sees.
|
|
223
|
+
if (page)
|
|
224
|
+
value = o.select ? (typeof page.value === "string" ? clip(page.value.trim()) : null) : found(re, page.text ?? "");
|
|
225
|
+
else if (o.path)
|
|
181
226
|
value = json === undefined ? null : (getPath(json, String(o.path)) ?? null);
|
|
227
|
+
else if (sel)
|
|
228
|
+
value = selected(body, sel, o.attr ? String(o.attr) : undefined);
|
|
182
229
|
// In what a reader sees first; a pattern that is about the markup finds it in the source.
|
|
183
230
|
else
|
|
184
|
-
value = (html ? found(re, pageText(
|
|
185
|
-
source.status(value === null ? `found nothing ${o.path ? `at ${o.path}` : "with the pattern"}` : `now: ${harmless(value)}`);
|
|
231
|
+
value = (html ? found(re, pageText(body)) : null) ?? found(re, body);
|
|
232
|
+
source.status(value === null ? `found nothing ${o.path ? `at ${o.path}` : o.select ? "with the selector" : "with the pattern"}` : `now: ${harmless(value)}`);
|
|
186
233
|
const was = source.changed(value);
|
|
234
|
+
if (answer)
|
|
235
|
+
keepSince(source, about, answer);
|
|
187
236
|
if (was)
|
|
188
|
-
source.emit("changed", { url: o.url, value: shown(value), previous: shown(was.previous) });
|
|
237
|
+
source.emit("changed", { url: o.url, value: shown(value), previous: shown(was.previous), ...numbers(value, was.previous) });
|
|
189
238
|
return;
|
|
190
239
|
}
|
|
191
|
-
const text = (html ? pageText(
|
|
240
|
+
const text = (page ? (page.text ?? "") : html ? pageText(body) : json !== undefined ? JSON.stringify(json, null, 1) : body).slice(0, 200_000);
|
|
192
241
|
source.status(`now: ${text.split("\n").length} lines`);
|
|
193
242
|
const was = source.changed(text);
|
|
243
|
+
if (answer)
|
|
244
|
+
keepSince(source, about, answer);
|
|
194
245
|
if (!was)
|
|
195
246
|
return;
|
|
196
247
|
const diff = lineDiff(String(was.previous), text);
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import type { SourceType } from "../source-types.ts";
|
|
2
|
+
import type { SourceApi } from "../sources.ts";
|
|
3
|
+
/**
|
|
4
|
+
* A mailbox: reports every new mail, the moment the server says there is one (IMAP IDLE).
|
|
5
|
+
*
|
|
6
|
+
* options: { host, user, password, port?, security?, mailbox?, from?, subject?, maxMb?, keepDays?,
|
|
7
|
+
* smtpHost?, smtpPort?, sendTo?, sender?, insecureTls?, local? }
|
|
8
|
+
* security "tls" (default, port 993), "starttls" (port 143), "none" only for a server on this machine or network
|
|
9
|
+
* mailbox which folder to watch ("INBOX")
|
|
10
|
+
* from only mail that says it is from one of these: an address, or "@domain"
|
|
11
|
+
* subject only mail whose subject this regular expression finds
|
|
12
|
+
* maxMb attachments are kept up to this size (10)
|
|
13
|
+
* sendTo addresses the agent may write to, through smtpHost. Without it nothing can be sent
|
|
14
|
+
* signIn "microsoft": no password. The owner signs in at Microsoft in a browser, once, with a
|
|
15
|
+
* code pine shows (the device code flow), and pine logs in with what it is given
|
|
16
|
+
* (XOAUTH2). For Outlook.com, Hotmail and Microsoft 365, which take no password over IMAP.
|
|
17
|
+
* Needs `clientId`: an app the owner registered at Microsoft; `tenant` says for which accounts.
|
|
18
|
+
*
|
|
19
|
+
* event "mail": { from, fromName?, to, subject, date?, text, attachments: [{ file, path?, type, size }], images?, cut? }
|
|
20
|
+
* It can send, to the addresses on `sendTo` only: { to, subject, text }
|
|
21
|
+
*
|
|
22
|
+
* It only reads: the mailbox is opened read-only (EXAMINE) and messages are fetched with PEEK, so
|
|
23
|
+
* nothing is marked as read, moved or deleted. Mail that is there when it first connects is the
|
|
24
|
+
* starting point and is not reported. Who a mail says it is from is what the sender wrote, and
|
|
25
|
+
* anyone can write any address: a mail is outside content, whoever it claims to come from.
|
|
26
|
+
*/
|
|
27
|
+
/** How long things take. A test makes them short. (`second`: what a second of waiting between two questions to Microsoft lasts.) */
|
|
28
|
+
export declare const pace: {
|
|
29
|
+
retry: number;
|
|
30
|
+
retryMax: number;
|
|
31
|
+
idle: number;
|
|
32
|
+
noop: number;
|
|
33
|
+
timeout: number;
|
|
34
|
+
second: number;
|
|
35
|
+
};
|
|
36
|
+
/**
|
|
37
|
+
* Where Microsoft signs people in. A test puts its stand-in here. It is not an option of the
|
|
38
|
+
* source: whoever names this address is handed the sign-in, so no agent and no configuration does.
|
|
39
|
+
*/
|
|
40
|
+
export declare const oauth: {
|
|
41
|
+
authority: string;
|
|
42
|
+
};
|
|
43
|
+
export declare const type: SourceType;
|
|
44
|
+
/** A message as it goes over the wire: headers of pine's making, the text as base64. */
|
|
45
|
+
export declare function composeMail(m: {
|
|
46
|
+
from: string;
|
|
47
|
+
to: string;
|
|
48
|
+
subject: string;
|
|
49
|
+
text: string;
|
|
50
|
+
}): string;
|
|
51
|
+
export default function mail(source: SourceApi): Promise<void>;
|