@pmndre/pine 0.22.1 → 0.33.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. package/README.md +2 -1
  2. package/changes.json +265 -0
  3. package/docs/SKILL.md +31 -7
  4. package/docs/references/agents.md +245 -21
  5. package/docs/references/coding.md +1 -1
  6. package/docs/references/computer.md +1 -1
  7. package/docs/references/configuration.md +14 -5
  8. package/docs/references/mcp.md +9 -0
  9. package/docs/references/operations.md +20 -9
  10. package/docs/references/overview.md +3 -1
  11. package/docs/references/security.md +83 -24
  12. package/docs/references/setup.md +55 -9
  13. package/docs/references/source-types.md +1418 -0
  14. package/docs/references/sources.md +333 -19
  15. package/docs/references/stack.md +1 -0
  16. package/docs/references/tui.md +86 -12
  17. package/examples/home/AGENTS.md +4 -0
  18. package/examples/home/TOOLS.md +5 -0
  19. package/examples/home/agents/main/workspace/AGENTS.md +1 -1
  20. package/examples/home/agents/main/workspace/TOOLS.md +5 -0
  21. package/mcp/browser.d.ts +6 -0
  22. package/mcp/browser.js +19 -0
  23. package/mcp/render.d.ts +1 -0
  24. package/mcp/render.js +237 -0
  25. package/package.json +3 -1
  26. package/src/actions.d.ts +6 -1
  27. package/src/actions.js +17 -4
  28. package/src/admin.d.ts +4 -0
  29. package/src/admin.js +25 -6
  30. package/src/agents.d.ts +167 -2
  31. package/src/agents.js +912 -45
  32. package/src/catalogue.d.ts +60 -0
  33. package/src/catalogue.js +791 -0
  34. package/src/cli.js +10 -7
  35. package/src/config.d.ts +53 -4
  36. package/src/config.js +30 -5
  37. package/src/goals.d.ts +1 -1
  38. package/src/goals.js +14 -1
  39. package/src/manual.js +24 -2
  40. package/src/mime.d.ts +78 -0
  41. package/src/mime.js +342 -0
  42. package/src/net.d.ts +11 -0
  43. package/src/net.js +29 -2
  44. package/src/pine.js +10 -1
  45. package/src/router.js +3 -3
  46. package/src/sandbox-linux.d.ts +3 -1
  47. package/src/sandbox-linux.js +5 -0
  48. package/src/security.d.ts +22 -6
  49. package/src/security.js +44 -13
  50. package/src/slots.d.ts +106 -0
  51. package/src/slots.js +250 -0
  52. package/src/source-types.d.ts +20 -0
  53. package/src/sources/calendar.d.ts +24 -0
  54. package/src/sources/calendar.js +265 -0
  55. package/src/sources/command.js +109 -4
  56. package/src/sources/discord.d.ts +4 -0
  57. package/src/sources/discord.js +298 -0
  58. package/src/sources/feed.d.ts +19 -0
  59. package/src/sources/feed.js +257 -0
  60. package/src/sources/http.js +154 -103
  61. package/src/sources/mail.d.ts +51 -0
  62. package/src/sources/mail.js +1013 -0
  63. package/src/sources/matrix.d.ts +4 -0
  64. package/src/sources/matrix.js +232 -0
  65. package/src/sources/modbus.d.ts +15 -0
  66. package/src/sources/modbus.js +240 -0
  67. package/src/sources/onvif.d.ts +35 -0
  68. package/src/sources/onvif.js +326 -0
  69. package/src/sources/presence.d.ts +80 -0
  70. package/src/sources/presence.js +480 -0
  71. package/src/sources/serial.d.ts +21 -0
  72. package/src/sources/serial.js +220 -0
  73. package/src/sources/slack.d.ts +16 -0
  74. package/src/sources/slack.js +1062 -0
  75. package/src/sources/smtp.d.ts +4 -0
  76. package/src/sources/smtp.js +360 -0
  77. package/src/sources/stream.d.ts +68 -0
  78. package/src/sources/stream.js +534 -0
  79. package/src/sources/syslog.d.ts +32 -0
  80. package/src/sources/syslog.js +243 -0
  81. package/src/sources/system.d.ts +14 -0
  82. package/src/sources/system.js +236 -0
  83. package/src/sources/system.swift +102 -0
  84. package/src/sources/telegram.js +34 -1
  85. package/src/sources.d.ts +41 -1
  86. package/src/sources.js +147 -24
  87. package/src/tools.d.ts +1 -1
  88. package/src/tools.js +304 -23
  89. package/src/tui/app.d.ts +35 -2
  90. package/src/tui/app.js +306 -43
  91. package/src/tui/levels.d.ts +68 -0
  92. package/src/tui/levels.js +273 -0
  93. package/src/tui/picture.d.ts +17 -1
  94. package/src/tui/picture.js +72 -5
  95. package/src/types.d.ts +3 -0
  96. package/src/update.d.ts +9 -1
  97. package/src/update.js +75 -3
  98. package/src/watching.d.ts +156 -0
  99. package/src/watching.js +609 -4
package/src/watching.js CHANGED
@@ -1,5 +1,14 @@
1
- // What the source types that look at something again and again (http, command) have in common:
2
- // picking the part that counts out of what they got, and saying what changed in it.
1
+ // What the source types that look at something again and again (http, feed, calendar, command)
2
+ // have in common: asking a web address, picking the part that counts out of what they got, and
3
+ // saying what changed in it.
4
+ import { execFile, spawn } from "node:child_process";
5
+ import crypto from "node:crypto";
6
+ import fs from "node:fs";
7
+ import path from "node:path";
8
+ import { fetch } from "undici";
9
+ import { EXT, PINE_ROOT } from "./config.js";
10
+ import { isLocalHost, publicOnlyAgent } from "./net.js";
11
+ import { errText, parseDuration } from "./util.js";
3
12
  const LINES = 30;
4
13
  const LINE = 300;
5
14
  const VALUE = 4000;
@@ -47,7 +56,11 @@ export function lineDiff(before, now) {
47
56
  const more = Math.max(0, added.length - LINES) + Math.max(0, removed.length - LINES);
48
57
  return { added: added.slice(0, LINES).map((l) => clip(l, LINE)), removed: removed.slice(0, LINES).map((l) => clip(l, LINE)), ...(more ? { more } : {}) };
49
58
  }
50
- const ENTITIES = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", nbsp: " ", euro: "€", pound: "£", copy: "©", ndash: "–", mdash: "—", hellip: "…" };
59
+ const ENTITIES = {
60
+ ...{ amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", nbsp: " ", euro: "€", pound: "£", copy: "©", ndash: "–", mdash: "—", hellip: "…" },
61
+ ...{ lsquo: "‘", rsquo: "’", ldquo: "“", rdquo: "”", sbquo: "‚", bdquo: "„", laquo: "«", raquo: "»", reg: "®", trade: "™", deg: "°", middot: "·", bull: "•", times: "×", yen: "¥", cent: "¢", sect: "§", shy: "", ensp: " ", emsp: " ", thinsp: " " },
62
+ ...{ auml: "ä", ouml: "ö", uuml: "ü", Auml: "Ä", Ouml: "Ö", Uuml: "Ü", szlig: "ß", eacute: "é", egrave: "è", ecirc: "ê", agrave: "à", aacute: "á", acirc: "â", ccedil: "ç", ntilde: "ñ", oacute: "ó", iacute: "í", uacute: "ú", aring: "å", oslash: "ø", aelig: "æ" },
63
+ };
51
64
  /** What a reader of a web page sees, line by line: no scripts, no styles, no markup. */
52
65
  export function pageText(html) {
53
66
  return html
@@ -57,7 +70,7 @@ export function pageText(html) {
57
70
  .replace(/<[^>]+>/g, " ")
58
71
  .replace(/&(#x?[0-9a-f]+|[a-z]+);/gi, (all, name) => {
59
72
  if (name[0] !== "#")
60
- return ENTITIES[name.toLowerCase()] ?? all;
73
+ return ENTITIES[name] ?? ENTITIES[name.toLowerCase()] ?? all;
61
74
  const code = name[1].toLowerCase() === "x" ? Number.parseInt(name.slice(2), 16) : Number.parseInt(name.slice(1), 10);
62
75
  return Number.isFinite(code) && code > 0 && code < 0x110000 ? String.fromCodePoint(code) : all;
63
76
  })
@@ -66,3 +79,595 @@ export function pageText(html) {
66
79
  .filter(Boolean)
67
80
  .join("\n");
68
81
  }
82
+ /**
83
+ * A cap for a source that is told things rather than looking: at most `perMinute` reports in any
84
+ * minute. What comes on top is dropped and counted, so that a device or a service that talks all
85
+ * day fills neither the event log nor a conversation. `dropped()` says how many were left out
86
+ * since it was last asked, for the next report to carry.
87
+ */
88
+ export function capped(perMinute) {
89
+ let from = 0;
90
+ let n = 0;
91
+ let left = 0;
92
+ return {
93
+ ok() {
94
+ const now = Date.now();
95
+ if (now - from >= 60_000) {
96
+ from = now;
97
+ n = 0;
98
+ }
99
+ if (n < perMinute)
100
+ return ++n > 0;
101
+ left++;
102
+ return false;
103
+ },
104
+ dropped() {
105
+ const d = left;
106
+ left = 0;
107
+ return d;
108
+ },
109
+ };
110
+ }
111
+ // ---- a value that is a number ---------------------------------------------------------------------
112
+ /** Signs and codes of currencies whose amounts are written the continental way: 1.299,00. */
113
+ const CONTINENTAL = /€|\bEUR\b|zł|\bPLN\b|Kč|\bCZK\b|\bkr\b|\bDKK\b|\bSEK\b|\bNOK\b/i;
114
+ /**
115
+ * The number a value stands for, if it reads as a number or an amount and nothing else:
116
+ * 12.5, "87 %", "$1,299.00", "1.299,00 €", "-3,5 °C". A text with more to it than a sign, a unit
117
+ * or a currency is not one, and neither is a version (1.2.3), a date or a time.
118
+ *
119
+ * Two separators: the last one is the decimal point. One kind several times: thousands. One
120
+ * separator once is the hard case. Followed by one, two, or more than three digits it is a decimal
121
+ * point. Followed by exactly three it could be either, and then the rest of the text decides: next
122
+ * to a continental currency (€, zł, kr, Kč) a dot groups thousands and a comma is the decimal
123
+ * point; anywhere else it is the other way round, as a program writes numbers.
124
+ */
125
+ export function numberOf(value) {
126
+ if (typeof value === "number")
127
+ return Number.isFinite(value) ? value : undefined;
128
+ if (typeof value !== "string" || value.length > 40)
129
+ return undefined;
130
+ const m = /^\s*([^\d\s+\-−.,]{0,4})\s*([+\-−]?)\s*(\d(?:[\d.,'’ \u00a0\u202f]*\d)?)\s*([^\d\s.,]{0,6})\s*$/.exec(value);
131
+ if (!m)
132
+ return undefined;
133
+ const [, before, sign, digits, after] = m;
134
+ // Spaces and apostrophes only ever group: 1 299,00 and 1'299.00.
135
+ const grouped = /['’ \u00a0\u202f]/.test(digits);
136
+ if (grouped && !/^\d{1,3}(?:['’ \u00a0\u202f]\d{3})+(?:[.,]\d+)?$/.test(digits))
137
+ return undefined;
138
+ const body = digits.replace(/['’ \u00a0\u202f]/g, "");
139
+ const dots = body.split(".").length - 1;
140
+ const commas = body.split(",").length - 1;
141
+ const groups = (sep) => new RegExp(`^\\d{1,3}(?:\\${sep}\\d{3})+$`);
142
+ let plain;
143
+ if (dots && commas) {
144
+ const point = body.lastIndexOf(".") > body.lastIndexOf(",") ? "." : ",";
145
+ const other = point === "." ? "," : ".";
146
+ const [whole, frac, ...more] = body.split(point);
147
+ if (more.length || frac.includes(other) || !groups(other).test(whole))
148
+ return undefined;
149
+ plain = `${whole.split(other).join("")}.${frac}`;
150
+ }
151
+ else if (dots + commas > 1) {
152
+ const sep = dots ? "." : ",";
153
+ if (!groups(sep).test(body))
154
+ return undefined;
155
+ plain = body.split(sep).join("");
156
+ }
157
+ else if (dots + commas === 1) {
158
+ const sep = dots ? "." : ",";
159
+ const [whole, frac] = body.split(sep);
160
+ const either = frac.length === 3 && whole.length <= 3 && whole !== "0" && !grouped;
161
+ const thousands = either && (CONTINENTAL.test(before + after) ? sep === "." : sep === ",");
162
+ plain = thousands ? whole + frac : `${whole}.${frac}`;
163
+ }
164
+ else
165
+ plain = body;
166
+ const n = Number(plain);
167
+ return Number.isFinite(n) ? (sign && sign !== "+" ? -n : n) : undefined;
168
+ }
169
+ /** `number` and `previousNumber` for an event, where the value now or the one before reads as a number. */
170
+ export function numbers(value, previous) {
171
+ const now = numberOf(value);
172
+ const was = numberOf(previous);
173
+ return { ...(now !== undefined ? { number: now } : {}), ...(was !== undefined ? { previousNumber: was } : {}) };
174
+ }
175
+ const VOID = new Set(["area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", "source", "track", "wbr"]);
176
+ const RAW = new Set(["script", "style", "textarea", "title", "noscript", "template"]);
177
+ /** Tags that end an open one of the same family without a closing tag, as browsers read them. */
178
+ const ENDS = { li: ["li"], dt: ["dt", "dd"], dd: ["dt", "dd"], tr: ["tr", "td", "th"], td: ["td", "th"], th: ["td", "th"], option: ["option"], p: ["p"] };
179
+ const WALLS = new Set(["ul", "ol", "dl", "table", "select"]);
180
+ const BLOCKS = new Set(["div", "ul", "ol", "table", "section", "article", "header", "footer", "main", "nav", "aside", "form", "h1", "h2", "h3", "h4", "h5", "h6", "blockquote", "pre", "hr", "dl", "figure"]);
181
+ /** The elements of a page in the order they begin, each with its place. Forgiving, as pages need it. */
182
+ function elements(html) {
183
+ const out = [];
184
+ const open = [];
185
+ const close = (upTo, at) => {
186
+ for (let i = open.length - 1; i >= upTo; i--)
187
+ open[i].to = at;
188
+ open.length = upTo;
189
+ };
190
+ const tagRe = /<!--[\s\S]*?-->|<!\[CDATA\[[\s\S]*?\]\]>|<![^>]*>|<\?[^>]*>|<(\/?)([A-Za-z][A-Za-z0-9:-]*)((?:[^>"']|"[^"]*"|'[^']*')*)>/g;
191
+ let m;
192
+ while ((m = tagRe.exec(html))) {
193
+ if (!m[2])
194
+ continue;
195
+ const tag = m[2].toLowerCase();
196
+ if (m[1]) {
197
+ const i = open.findLastIndex((e) => e.tag === tag);
198
+ if (i >= 0)
199
+ close(i, m.index);
200
+ continue;
201
+ }
202
+ // An opening tag ends what cannot contain it: a list item the one before, a block an open paragraph.
203
+ // Never through the list or table it stands in, so that one nested inside another stays whole.
204
+ const enders = ENDS[tag] ?? (BLOCKS.has(tag) ? ["p"] : undefined);
205
+ if (enders) {
206
+ const wall = open.findLastIndex((e) => WALLS.has(e.tag));
207
+ const i = open.findIndex((e, at) => at > wall && enders.includes(e.tag));
208
+ if (i >= 0)
209
+ close(i, m.index);
210
+ }
211
+ const attrs = {};
212
+ const attrRe = /([^\s"'<>\/=]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+)))?/g;
213
+ let a;
214
+ while ((a = attrRe.exec(m[3])))
215
+ attrs[a[1].toLowerCase()] ??= unescapeText(a[2] ?? a[3] ?? a[4] ?? "");
216
+ const end = m.index + m[0].length;
217
+ const el = { tag, attrs, parent: open.at(-1), from: end, to: end };
218
+ out.push(el);
219
+ if (VOID.has(tag) || /\/\s*$/.test(m[3]))
220
+ continue;
221
+ if (RAW.has(tag)) {
222
+ const stop = html.toLowerCase().indexOf(`</${tag}`, end);
223
+ el.to = stop < 0 ? html.length : stop;
224
+ tagRe.lastIndex = el.to;
225
+ continue;
226
+ }
227
+ open.push(el);
228
+ }
229
+ close(0, html.length);
230
+ return out;
231
+ }
232
+ /** A text with its character references (&amp;, &#8364;, &#x41;) written out. */
233
+ export const unescaped = (s) => unescapeText(s);
234
+ const unescapeText = (s) => s.replace(/&(#x?[0-9a-f]+|[a-z]+);/gi, (all, name) => {
235
+ if (name[0] !== "#")
236
+ return ENTITIES[name] ?? ENTITIES[name.toLowerCase()] ?? all;
237
+ const code = name[1].toLowerCase() === "x" ? Number.parseInt(name.slice(2), 16) : Number.parseInt(name.slice(1), 10);
238
+ return Number.isFinite(code) && code > 0 && code < 0x110000 ? String.fromCodePoint(code) : all;
239
+ });
240
+ const SELECT_HELP = 'It takes a tag, #id, .class, [attribute] and [attribute=value], put together (div.price, span#total) and nested with a space (inside) or > (directly inside), for example ".product .price" or "table#stock > tbody td.count". Nothing with a colon (:first-child, :not) and no + or ~';
241
+ /** A CSS selector as this file can read it, or an error that says what it takes. */
242
+ export function selector(source) {
243
+ const text = String(source ?? "").trim();
244
+ if (!text)
245
+ throw new Error(`"select" is empty. ${SELECT_HELP}`);
246
+ if (text.length > 300)
247
+ throw new Error('"select" is too long: 300 characters at most');
248
+ const bad = (what) => new Error(`"select" cannot be read here (${what}). ${SELECT_HELP}`);
249
+ const chains = [];
250
+ // Commas separate alternatives, except inside brackets.
251
+ for (const alt of text.split(/,(?![^\[]*\])/)) {
252
+ const chain = [];
253
+ let rest = alt.trim();
254
+ let how = " ";
255
+ if (!rest)
256
+ throw bad("an empty part between commas");
257
+ while (rest) {
258
+ const part = { classes: [], attrs: [] };
259
+ let any = false;
260
+ let m;
261
+ if ((m = /^(\*|[A-Za-z][A-Za-z0-9-]*)/.exec(rest))) {
262
+ if (m[1] !== "*")
263
+ part.tag = m[1].toLowerCase();
264
+ rest = rest.slice(m[0].length);
265
+ any = true;
266
+ }
267
+ while (rest && !/^[\s>]/.test(rest)) {
268
+ if ((m = /^#([\w-]+)/.exec(rest)))
269
+ part.id = m[1];
270
+ else if ((m = /^\.([\w-]+)/.exec(rest)))
271
+ part.classes.push(m[1]);
272
+ else if ((m = /^\[\s*([\w:-]+)\s*(?:([~^$*|]?=)\s*(?:"([^"]*)"|'([^']*)'|([^\]\s]*)))?\s*\]/.exec(rest)))
273
+ part.attrs.push({ name: m[1].toLowerCase(), op: m[2], value: m[2] ? (m[3] ?? m[4] ?? m[5] ?? "") : undefined });
274
+ else
275
+ throw bad(rest[0] === ":" ? "a part with a colon" : rest[0] === "+" || rest[0] === "~" ? `the sign ${rest[0]}` : `at "${rest.slice(0, 20)}"`);
276
+ rest = rest.slice(m[0].length);
277
+ any = true;
278
+ }
279
+ if (!any)
280
+ throw bad(`at "${rest.slice(0, 20)}"`);
281
+ chain.push({ part, how });
282
+ const gap = /^\s*(>?)\s*/.exec(rest);
283
+ how = gap[1] ? ">" : " ";
284
+ rest = rest.slice(gap[0].length);
285
+ if (!rest && gap[1])
286
+ throw bad("it ends with >");
287
+ }
288
+ chains.push(chain);
289
+ }
290
+ return chains;
291
+ }
292
+ const fits = (el, s) => {
293
+ if (s.tag && el.tag !== s.tag)
294
+ return false;
295
+ if (s.id !== undefined && el.attrs.id !== s.id)
296
+ return false;
297
+ if (s.classes.length) {
298
+ const have = (el.attrs.class ?? "").split(/\s+/);
299
+ if (!s.classes.every((c) => have.includes(c)))
300
+ return false;
301
+ }
302
+ return s.attrs.every(({ name, op, value }) => {
303
+ const v = el.attrs[name];
304
+ if (v === undefined)
305
+ return false;
306
+ if (!op)
307
+ return true;
308
+ const want = value ?? "";
309
+ return op === "=" ? v === want : op === "~=" ? v.split(/\s+/).includes(want) : op === "^=" ? want !== "" && v.startsWith(want) : op === "$=" ? want !== "" && v.endsWith(want) : op === "*=" ? want !== "" && v.includes(want) : v === want || v.startsWith(`${want}-`);
310
+ });
311
+ };
312
+ const hangs = (el, chain, at) => {
313
+ if (!fits(el, chain[at].part))
314
+ return false;
315
+ if (at === 0)
316
+ return true;
317
+ if (chain[at].how === ">")
318
+ return el.parent !== undefined && hangs(el.parent, chain, at - 1);
319
+ for (let up = el.parent; up; up = up.parent)
320
+ if (hangs(up, chain, at - 1))
321
+ return true;
322
+ return false;
323
+ };
324
+ /**
325
+ * What the first element of a page that a selector matches says to a reader: its text, or with
326
+ * `attr` the value of that attribute (a price in a meta tag, a data- attribute). Nothing matched is null.
327
+ */
328
+ export function selected(html, chains, attr) {
329
+ const el = elements(html).find((e) => chains.some((chain) => hangs(e, chain, chain.length - 1)));
330
+ if (!el)
331
+ return null;
332
+ if (attr)
333
+ return el.attrs[attr.toLowerCase()] === undefined ? null : clip(el.attrs[attr.toLowerCase()].trim());
334
+ return clip(pageText(html.slice(el.from, el.to)).trim());
335
+ }
336
+ // ---- asking a web address -------------------------------------------------------------------------
337
+ /** How long one look at an address may take. */
338
+ export const LOOK_TIMEOUT = 20_000;
339
+ const MAX_BODY = 1024 * 1024;
340
+ /** Headers that carry a secret. Those go into `token`, which is kept out of the configuration. */
341
+ export const SECRET_HEADERS = ["authorization", "proxy-authorization", "cookie"];
342
+ /**
343
+ * Whether an address says by itself that it is on this computer or the local network. Such an
344
+ * address the owner confirmed as what it is. Any other one must stay on the public internet: a
345
+ * name that one day points into the local network is not followed there.
346
+ */
347
+ export const isLocalAddress = (url) => isLocalHost(url.hostname);
348
+ /** Why a look failed, in words: what lies behind "fetch failed". */
349
+ export const whyFailed = (e, url) => {
350
+ const text = errText(e?.cause ?? e);
351
+ return /resolves to a private address/.test(text) ? `${url.hostname} leads to an address on your own network. If that is meant, set the source up again with "local": true` : text;
352
+ };
353
+ /** The connections an address is asked over: as they are for a local one, kept on the public internet for any other. */
354
+ export const guardFor = (url, local) => (local === true || isLocalAddress(url) ? undefined : publicOnlyAgent(LOOK_TIMEOUT));
355
+ /**
356
+ * What every type that asks a web address checks when it is set up: that it is one, that nothing
357
+ * secret is written into it or into a header, and how often it may be asked. Returns the address.
358
+ */
359
+ export function askable(o, least, usual) {
360
+ let url;
361
+ try {
362
+ url = new URL(String(o.url));
363
+ }
364
+ catch {
365
+ throw new Error(`"${o.url}" is not an address. It looks like https://example.com/page`);
366
+ }
367
+ if (!["http:", "https:"].includes(url.protocol))
368
+ throw new Error("the address must start with https:// or http://");
369
+ if (url.username || url.password)
370
+ throw new Error('an address with a user name or password in it is not taken: leave it out of the address and name "token" in "ask"');
371
+ const every = o.every ?? usual;
372
+ if (parseDuration(every) < parseDuration(least))
373
+ throw new Error(`"every" must be at least "${least}"`);
374
+ const secret = Object.keys(o.headers ?? {}).filter((h) => SECRET_HEADERS.includes(h.toLowerCase()));
375
+ if (secret.length)
376
+ throw new Error(`the header ${secret.join(", ")} carries a secret: leave it out, name "token" in "ask" and give "tokenHeader": "${secret[0].toLowerCase()}"`);
377
+ if (o.tokenHeader && !/^[A-Za-z0-9-]+$/.test(o.tokenHeader))
378
+ throw new Error('"tokenHeader" is the name of a header, e.g. "x-api-key"');
379
+ return { url, every, local: o.local === true || isLocalAddress(url) };
380
+ }
381
+ /**
382
+ * One look: the answer's text, at most `maxBody` of it (1 MB). A redirect is followed only on the
383
+ * same host. With `since`, an address that has nothing new says so in one line instead of
384
+ * sending everything again.
385
+ */
386
+ export async function ask(start, o, guard, signal, since, maxBody = MAX_BODY) {
387
+ const headers = { ...(o.headers ?? {}) };
388
+ const has = (name) => Object.keys(headers).some((h) => h.toLowerCase() === name);
389
+ if (o.accept && !has("accept"))
390
+ headers.accept = o.accept;
391
+ if (o.token)
392
+ headers[o.tokenHeader ?? "authorization"] = o.tokenHeader ? o.token : `Bearer ${o.token}`;
393
+ let method = String(o.method ?? "GET").toUpperCase() === "POST" ? "POST" : "GET";
394
+ const body = method === "POST" ? String(o.body ?? "") : undefined;
395
+ if (body && !has("content-type"))
396
+ headers["content-type"] = /^\s*[{[]/.test(body) ? "application/json" : "text/plain; charset=utf-8";
397
+ // Only a plain look can be answered with "nothing new".
398
+ const conditional = method === "GET" && Boolean(since?.etag || since?.modified);
399
+ if (conditional && since?.etag)
400
+ headers["if-none-match"] = since.etag;
401
+ if (conditional && since?.modified)
402
+ headers["if-modified-since"] = since.modified;
403
+ let at = start;
404
+ for (let hop = 0; hop < 6; hop++) {
405
+ const res = await fetch(at, { method, headers, ...(method === "POST" ? { body } : {}), redirect: "manual", signal: AbortSignal.any([signal, AbortSignal.timeout(LOOK_TIMEOUT)]), ...(guard ? { dispatcher: guard } : {}) });
406
+ const to = res.status >= 300 && res.status < 400 && res.status !== 304 ? res.headers.get("location") : null;
407
+ if (to) {
408
+ await res.body?.cancel().catch(() => { });
409
+ const next = new URL(to, at);
410
+ // The owner confirmed one host. Another one is another decision, and must not get the token.
411
+ const upgrade = at.protocol === "http:" && next.protocol === "https:";
412
+ if (next.hostname !== start.hostname || (next.protocol !== at.protocol && !upgrade) || (next.port !== at.port && !upgrade))
413
+ throw new Error(`it sends on to ${next.host}, which is not the address that was confirmed. If that is the right one, set the source up with it`);
414
+ // Sent on after a POST: asked again the same way only where the answer says so (307, 308), as a browser does.
415
+ if (method === "POST" && res.status !== 307 && res.status !== 308) {
416
+ method = "GET";
417
+ delete headers["content-type"];
418
+ }
419
+ at = next;
420
+ continue;
421
+ }
422
+ if (res.status === 304 && conditional) {
423
+ await res.body?.cancel().catch(() => { });
424
+ return { type: "", body: "", unchanged: true, ...since };
425
+ }
426
+ if (!res.ok) {
427
+ await res.body?.cancel().catch(() => { });
428
+ throw new Error(`it answered ${res.status}`);
429
+ }
430
+ const chunks = [];
431
+ let size = 0;
432
+ const reader = res.body?.getReader();
433
+ while (reader) {
434
+ const { done, value } = await reader.read();
435
+ if (done)
436
+ break;
437
+ chunks.push(Buffer.from(value.subarray(0, maxBody - size)));
438
+ size += value.length;
439
+ if (size >= maxBody) {
440
+ await reader.cancel().catch(() => { });
441
+ break;
442
+ }
443
+ }
444
+ return { type: res.headers.get("content-type") ?? "", body: Buffer.concat(chunks).toString("utf8"), ...(size > maxBody ? { truncated: true } : {}), etag: res.headers.get("etag") ?? undefined, modified: res.headers.get("last-modified") ?? undefined };
445
+ }
446
+ throw new Error("it sends on and on (more than five redirects)");
447
+ }
448
+ /**
449
+ * What the address said about the answer a source last worked through, for the next look to ask
450
+ * with. `about` names what the source makes of an answer (the address, what it watches in it): a
451
+ * source that was set up anew for something else starts from a full answer.
452
+ */
453
+ export function sinceOf(source, about) {
454
+ const kept = source.kv.get("since");
455
+ return kept && kept.about === mark(about) && (kept.etag || kept.modified) ? { etag: kept.etag, modified: kept.modified } : undefined;
456
+ }
457
+ /** Remember what an answer said about itself, once the source has worked it through: a look that failed half way is made in full again. */
458
+ export function keepSince(source, about, answer) {
459
+ const had = source.kv.get("since");
460
+ if (!answer.etag && !answer.modified && !had)
461
+ return;
462
+ source.kv.set("since", { about: mark(about), etag: answer.etag, modified: answer.modified });
463
+ }
464
+ const mark = (about) => crypto.createHash("sha256").update(JSON.stringify(about)).digest("hex").slice(0, 16);
465
+ // ---- a page as a browser shows it -----------------------------------------------------------------
466
+ /** How long one look with the browser may take: starting it, loading the page, waiting for it to be quiet. */
467
+ export const RENDER_TIMEOUT = 45_000;
468
+ /** The most text a rendered page hands back, and the longest answer of the helper that is read. */
469
+ const RENDER_TEXT = 200_000;
470
+ const RENDER_ANSWER = 2 * 1024 * 1024;
471
+ /** How many browsers run at a time, for all sources together. A look that finds them busy waits its turn. */
472
+ const RENDER_AT_ONCE = 2;
473
+ /**
474
+ * For tests: a program that speaks the one-line protocol of the helper (mcp/render.ts) in its
475
+ * place, so that a rendered look can be tried without a browser, and a shorter time limit. Not an
476
+ * option of a source: a source's options are what an agent may set, and which program the daemon
477
+ * runs is not among them.
478
+ */
479
+ export const rendering = {};
480
+ /** Whether the address a look ended on is the one that was confirmed: the same host, and the same port unless http became https. */
481
+ export function sameSite(start, end) {
482
+ const upgrade = start.protocol === "http:" && end.protocol === "https:";
483
+ return end.hostname === start.hostname && (end.protocol === start.protocol || upgrade) && (end.port === start.port || upgrade);
484
+ }
485
+ /** A page's text as `pageText` gives it: line by line, spaces drawn together, no empty lines. */
486
+ const readerLines = (text) => text
487
+ .split("\n")
488
+ .map((l) => l.replace(/\s+/g, " ").trim())
489
+ .filter(Boolean)
490
+ .join("\n");
491
+ let renderBusy = 0;
492
+ const renderQueue = [];
493
+ /** Waits until a browser may be started. Resolves with what gives the turn back. */
494
+ function renderTurn(signal) {
495
+ return new Promise((resolve, reject) => {
496
+ const go = () => {
497
+ signal.removeEventListener("abort", gone);
498
+ renderBusy++;
499
+ let given = false;
500
+ resolve(() => {
501
+ if (given)
502
+ return;
503
+ given = true;
504
+ renderBusy--;
505
+ renderQueue.shift()?.();
506
+ });
507
+ };
508
+ const gone = () => {
509
+ renderQueue.splice(renderQueue.indexOf(go), 1);
510
+ reject(new Error("stopped"));
511
+ };
512
+ if (signal.aborted)
513
+ return reject(new Error("stopped"));
514
+ if (renderBusy < RENDER_AT_ONCE)
515
+ return go();
516
+ renderQueue.push(go);
517
+ signal.addEventListener("abort", gone, { once: true });
518
+ });
519
+ }
520
+ /**
521
+ * Ends whatever still runs out of a look's folder. The helper ends its browser when it ends, but a
522
+ * helper that had to be killed outright cannot, and the browser does not end by itself then. The
523
+ * browser is found by the folder its profile lies in, which is this look's and nobody else's.
524
+ */
525
+ function sweep(dir) {
526
+ return new Promise((resolve) => {
527
+ execFile("ps", ["axww", "-o", "pid=,args="], { maxBuffer: 16 * 1024 * 1024, timeout: 5000, env: { PATH: process.env.PATH ?? "/usr/bin:/bin" } }, (_err, out) => {
528
+ for (const line of String(out ?? "").split("\n")) {
529
+ const m = /^\s*(\d+)\s+(.*)$/.exec(line);
530
+ if (!m || !m[2].includes(dir + path.sep) || Number(m[1]) === process.pid)
531
+ continue;
532
+ try {
533
+ process.kill(Number(m[1]), "SIGKILL");
534
+ }
535
+ catch {
536
+ // gone already
537
+ }
538
+ }
539
+ resolve();
540
+ });
541
+ });
542
+ }
543
+ /**
544
+ * One look at a page with the browser pine ships: the page is loaded without a window, its
545
+ * scripts run, and what a reader sees then is handed back. The browser runs in a helper program
546
+ * (mcp/render.ts) that is started for this look and gone after it, whatever happens: an answer,
547
+ * no answer within the time limit, or the source stopping.
548
+ *
549
+ * `dir` is the source's own folder in pine's data. The helper works in a folder inside it, which
550
+ * is emptied afterwards. It gets no part of the daemon's environment, one line with the job, and
551
+ * is read one line of answer from, of bounded length. Throws with a sentence when the look failed.
552
+ */
553
+ export async function rendered(job, dir, signal) {
554
+ const done = await renderTurn(signal);
555
+ const scratch = path.join(dir, "browser");
556
+ try {
557
+ fs.rmSync(scratch, { recursive: true, force: true, maxRetries: 3 });
558
+ fs.mkdirSync(scratch, { recursive: true, mode: 0o700 });
559
+ const answer = await renderOnce(job, scratch, signal);
560
+ const start = new URL(job.url);
561
+ let final;
562
+ try {
563
+ final = new URL(String(answer.final));
564
+ }
565
+ catch {
566
+ throw new Error("its browser gave an answer that cannot be read");
567
+ }
568
+ if (!sameSite(start, final))
569
+ throw new Error(`it sends on to ${final.host}, which is not the address that was confirmed. If that is the right one, set the source up with it`);
570
+ if (job.select !== undefined)
571
+ return { value: typeof answer.value === "string" ? readerLines(answer.value.slice(0, RENDER_TEXT)) : null, final: final.href };
572
+ if (typeof answer.text !== "string")
573
+ throw new Error("its browser gave an answer that cannot be read");
574
+ return { text: readerLines(answer.text.slice(0, RENDER_TEXT)), final: final.href };
575
+ }
576
+ finally {
577
+ // What the browser left behind means it may still be there.
578
+ let left = [];
579
+ try {
580
+ left = fs.readdirSync(scratch);
581
+ }
582
+ catch {
583
+ // no folder, nothing left
584
+ }
585
+ if (left.length)
586
+ await sweep(scratch);
587
+ try {
588
+ fs.rmSync(scratch, { recursive: true, force: true, maxRetries: 3 });
589
+ }
590
+ catch {
591
+ // the next look empties it again
592
+ }
593
+ done();
594
+ }
595
+ }
596
+ /** Starts the helper, hands it the job, and reads its one line. Resolves or rejects only when the helper is gone. */
597
+ function renderOnce(job, scratch, signal) {
598
+ const limit = rendering.timeoutMs ?? RENDER_TIMEOUT;
599
+ const program = rendering.program ?? [process.execPath, path.join(PINE_ROOT, "mcp", `render${EXT}`)];
600
+ return new Promise((resolve, reject) => {
601
+ if (signal.aborted)
602
+ return reject(new Error("stopped"));
603
+ // Of the daemon's environment only what a browser needs to be found and to start. No secret is in it.
604
+ const env = { TMPDIR: scratch };
605
+ for (const k of ["HOME", "PATH", "PLAYWRIGHT_BROWSERS_PATH"])
606
+ if (process.env[k])
607
+ env[k] = process.env[k];
608
+ const child = spawn(program[0], program.slice(1), { cwd: scratch, env, stdio: ["pipe", "pipe", "ignore"], detached: true });
609
+ let out = Buffer.alloc(0);
610
+ let failure;
611
+ let hard;
612
+ // The whole group: a helper that started others must not leave them behind.
613
+ const group = (sig) => {
614
+ try {
615
+ process.kill(-child.pid, sig);
616
+ }
617
+ catch {
618
+ // nobody left in it
619
+ }
620
+ };
621
+ // Asked to end first, which lets the helper end its browser; a second later ended, whatever it is doing.
622
+ const end = (why) => {
623
+ failure ??= why;
624
+ if (hard || !child.pid)
625
+ return;
626
+ group("SIGTERM");
627
+ hard = setTimeout(() => group("SIGKILL"), 1000);
628
+ };
629
+ // The helper gives up by itself at `limit`, with a sentence. This is for a helper that does not.
630
+ const timer = setTimeout(() => end(new Error(`the page did not finish loading within ${Math.round(limit / 100) / 10} s`)), limit + Math.min(3000, limit));
631
+ const stopped = () => end(new Error("stopped"));
632
+ signal.addEventListener("abort", stopped, { once: true });
633
+ const settle = () => {
634
+ clearTimeout(timer);
635
+ clearTimeout(hard);
636
+ signal.removeEventListener("abort", stopped);
637
+ // What was ended may have left others in its group that did not listen.
638
+ if (hard)
639
+ group("SIGKILL");
640
+ if (failure)
641
+ return reject(failure);
642
+ let answer;
643
+ try {
644
+ answer = JSON.parse(out.toString("utf8").split("\n")[0]);
645
+ }
646
+ catch {
647
+ // not an answer
648
+ }
649
+ if (!answer || typeof answer !== "object" || Array.isArray(answer))
650
+ return reject(new Error(out.length ? "its browser gave an answer that cannot be read" : "its browser ended without an answer"));
651
+ // The helper's own sentence, never a page's: one line of it.
652
+ if (answer.ok !== true)
653
+ return reject(new Error(typeof answer.error === "string" && answer.error.trim() ? clip(answer.error.replace(/\s+/g, " ").trim(), 300) : "its browser gave an answer that cannot be read"));
654
+ resolve(answer);
655
+ };
656
+ child.stdout.on("data", (d) => {
657
+ if (failure)
658
+ return;
659
+ out = Buffer.concat([out, d]);
660
+ if (out.length > RENDER_ANSWER)
661
+ end(new Error("its browser answered with more than a page's text can be"));
662
+ });
663
+ child.stdin.on("error", () => { });
664
+ child.on("error", (e) => {
665
+ failure ??= new Error(`pine's browser could not be started (${errText(e)})`);
666
+ // A program that never started never closes.
667
+ if (!child.pid)
668
+ settle();
669
+ });
670
+ child.on("close", settle);
671
+ child.stdin.end(`${JSON.stringify({ url: job.url, ...(job.select !== undefined ? { select: job.select } : {}), ...(job.attr !== undefined ? { attr: job.attr } : {}), timeout: limit, ...(job.localFile ? { localFile: job.localFile } : {}) })}\n`);
672
+ });
673
+ }