@parcha/pi-browser 0.1.0-beta.28 → 0.1.0-beta.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +20 -0
- package/README.md +22 -7
- package/dist/coding-agent-local.d.ts +2 -1
- package/dist/coding-agent-local.js +5 -3
- package/dist/core/contract.js +4 -3
- package/dist/core/entities.d.ts +3 -0
- package/dist/core/entities.js +68 -0
- package/dist/core/failures.d.ts +2 -0
- package/dist/core/failures.js +8 -1
- package/dist/core/guard.d.ts +3 -0
- package/dist/core/guard.js +5 -0
- package/dist/core/host.d.ts +13 -0
- package/dist/core/page-classifier.d.ts +77 -0
- package/dist/core/page-classifier.js +92 -0
- package/dist/core/page-markdown.d.ts +5 -0
- package/dist/core/page-markdown.js +151 -0
- package/dist/core/page-scan.d.ts +95 -0
- package/dist/core/page-scan.js +542 -0
- package/dist/core/public-fetch.d.ts +0 -7
- package/dist/core/public-fetch.js +0 -25
- package/dist/core/read-page-worker.d.ts +8 -0
- package/dist/core/read-page-worker.js +32 -0
- package/dist/core/read-page.d.ts +46 -0
- package/dist/core/read-page.js +96 -0
- package/dist/core/tools.d.ts +9 -2
- package/dist/core/tools.js +56 -31
- package/dist/core/web.d.ts +2 -0
- package/dist/core/web.js +15 -7
- package/dist/durable/store.d.ts +7 -1
- package/dist/durable/store.js +70 -20
- package/dist/durable.d.ts +10 -2
- package/dist/durable.js +29 -7
- package/dist/index.d.ts +3 -1
- package/dist/index.js +3 -1
- package/dist/providers/browserbase.js +7 -2
- package/dist/providers/kernel.js +7 -2
- package/package.json +1 -1
|
@@ -0,0 +1,542 @@
|
|
|
1
|
+
// The readings of an HTML page that need no DOM: the page without the blocks that hold no readable text, the page cut
|
|
2
|
+
// to a length, and one pass over its tags for its title, its links and its plain text. Each is linear in the page's
|
|
3
|
+
// length whatever its markup (a page with no ">" after a "<", a thousand unclosed scripts): a scan with indexOf, or a
|
|
4
|
+
// pattern whose every try is bounded by the next character it can stop at. read-page.ts runs them on the event loop;
|
|
5
|
+
// its worker runs the scan for the title and links.
|
|
6
|
+
const INERT_NAMES = new Set(["script", "style", "svg", "noscript", "template"]);
|
|
7
|
+
/** The elements whose content is text, not markup, to their first end tag, as a browser reads them: a tag inside one is
|
|
8
|
+
* none (script, style and noscript are also inert). */
|
|
9
|
+
const TEXT_ONLY = new Set(["script", "style", "noscript", "textarea", "title", "xmp", "iframe", "noembed", "noframes"]);
|
|
10
|
+
/** The kept ones whose text the drop also writes as text: each "<" in a textarea's or a title's text becomes "<", so
|
|
11
|
+
* a parser that reads them as markup (node-html-parser does) reads the text a browser shows. */
|
|
12
|
+
const KEPT_TEXT = new Set(["textarea", "title"]);
|
|
13
|
+
const CLOSER = Object.fromEntries([...TEXT_ONLY, "svg", "template"].map((name) => [name, new RegExp(`</${name}(?=[\\s/>])[^>]*>?`, "gi")]));
|
|
14
|
+
/** A page without its script, style, svg, noscript and template elements and its comments, in steps so a caller on the
|
|
15
|
+
* event loop can yield between them (dropInert, all at once). One whose end is missing runs to the end of the page, as
|
|
16
|
+
* a browser reads it; but an svg's start tag goes alone, since a browser lets the HTML that follows out of an unclosed
|
|
17
|
+
* svg. A template ends at the end tag that closes it, as in a browser: a template inside it is counted, so its end tag
|
|
18
|
+
* ends only the inner one, and the text of an element that holds text, not markup, is never read as tags in it
|
|
19
|
+
* (TEXT_ONLY: a script, a textarea, a title). Out of a template, a textarea's or a title's text is not read as tags
|
|
20
|
+
* either: a "<template>" or a "<script>" in it is text, kept with each "<" written "<" (KEPT_TEXT). A nested svg ends at the first end tag of its name. A search for an end tag that finds none is not made again from
|
|
21
|
+
* a later start of that name (none follows it either), so a page of unclosed svgs is read once, not once for each; and
|
|
22
|
+
* the templates' ends are found in one forward pass however many there are (#templateEnd). */
|
|
23
|
+
export class InertDrop {
|
|
24
|
+
html;
|
|
25
|
+
// Its blocks are found among the page's tags (tagsOf), so a "<script>" inside an attribute value is none.
|
|
26
|
+
#next;
|
|
27
|
+
#unclosed = new Map();
|
|
28
|
+
#kept = [];
|
|
29
|
+
/** Where the scan is, and where the HTML kept since the last block dropped starts. */
|
|
30
|
+
#at = 0;
|
|
31
|
+
#from = 0;
|
|
32
|
+
#done = false;
|
|
33
|
+
#templates = null;
|
|
34
|
+
constructor(html) {
|
|
35
|
+
this.html = html;
|
|
36
|
+
this.#next = tagsOf(html);
|
|
37
|
+
}
|
|
38
|
+
/** Drops the blocks found in the next `chars` characters or so; true once the whole page is read. A template's end
|
|
39
|
+
* is looked for `chars` characters at a time too, so no step reads much more than that. */
|
|
40
|
+
step(chars) {
|
|
41
|
+
const { html } = this;
|
|
42
|
+
const until = this.#at + chars;
|
|
43
|
+
while (!this.#done && this.#at < until) {
|
|
44
|
+
const tag = this.#next(this.#at);
|
|
45
|
+
if (!tag) {
|
|
46
|
+
this.#finish();
|
|
47
|
+
break;
|
|
48
|
+
}
|
|
49
|
+
const comment = html.startsWith("!--", tag.lt + 1);
|
|
50
|
+
// A tag with no ">" after it: the rest of the page is kept as it is (a comment with no "-->" runs to its end).
|
|
51
|
+
if (!comment && tag.gt < 0) {
|
|
52
|
+
this.#finish();
|
|
53
|
+
break;
|
|
54
|
+
}
|
|
55
|
+
if (!comment && (tag.end || !INERT_NAMES.has(tag.name))) {
|
|
56
|
+
if (tag.end || !KEPT_TEXT.has(tag.name)) {
|
|
57
|
+
this.#at = tag.gt + 1;
|
|
58
|
+
continue;
|
|
59
|
+
}
|
|
60
|
+
// A textarea or a title: its text is passed to its end tag, never read as tags, so a "<script>" or a
|
|
61
|
+
// "<template>" in it drops nothing, and is kept with each "<" written "<"; with no end tag the rest of the
|
|
62
|
+
// page is its text.
|
|
63
|
+
const close = this.#endTag(tag.name, tag.gt + 1);
|
|
64
|
+
const text = html.slice(tag.gt + 1, close ? close.start : html.length);
|
|
65
|
+
if (text.includes("<")) {
|
|
66
|
+
this.#kept.push(html.slice(this.#from, tag.gt + 1), text.replaceAll("<", "<"));
|
|
67
|
+
this.#from = close ? close.start : html.length;
|
|
68
|
+
}
|
|
69
|
+
if (!close) {
|
|
70
|
+
this.#finish();
|
|
71
|
+
break;
|
|
72
|
+
}
|
|
73
|
+
this.#at = close.end;
|
|
74
|
+
continue;
|
|
75
|
+
}
|
|
76
|
+
let end = tag.gt < 0 ? html.length : tag.gt + 1;
|
|
77
|
+
if (!comment) {
|
|
78
|
+
const name = tag.name;
|
|
79
|
+
if (name === "template") {
|
|
80
|
+
const close = this.#templateEnd(tag.lt, end, chars);
|
|
81
|
+
// Not found within this step's reading: the next step goes on from this template.
|
|
82
|
+
if (close === undefined)
|
|
83
|
+
break;
|
|
84
|
+
end = close >= 0 ? close : html.length;
|
|
85
|
+
}
|
|
86
|
+
else {
|
|
87
|
+
const close = this.#endTagEnd(name, end);
|
|
88
|
+
end = close >= 0 ? close : name === "svg" ? end : html.length;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
this.#kept.push(html.slice(this.#from, tag.lt));
|
|
92
|
+
this.#from = this.#at = end;
|
|
93
|
+
if (end >= html.length)
|
|
94
|
+
this.#finish();
|
|
95
|
+
}
|
|
96
|
+
return this.#done;
|
|
97
|
+
}
|
|
98
|
+
/** The first end tag named `name` from `from` (CLOSER), where it starts and the offset just past it; null when there
|
|
99
|
+
* is none. */
|
|
100
|
+
#endTag(name, from) {
|
|
101
|
+
if (from >= (this.#unclosed.get(name) ?? Number.POSITIVE_INFINITY))
|
|
102
|
+
return null;
|
|
103
|
+
const closer = CLOSER[name];
|
|
104
|
+
closer.lastIndex = from;
|
|
105
|
+
const close = closer.exec(this.html);
|
|
106
|
+
if (!close) {
|
|
107
|
+
this.#unclosed.set(name, from);
|
|
108
|
+
return null;
|
|
109
|
+
}
|
|
110
|
+
return { start: close.index, end: close.index + close[0].length };
|
|
111
|
+
}
|
|
112
|
+
/** The offset just past the first end tag named `name` from `from`, or -1 when there is none (#endTag). */
|
|
113
|
+
#endTagEnd(name, from) {
|
|
114
|
+
return this.#endTag(name, from)?.end ?? -1;
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* The offset just past the end tag that closes the template whose start tag is at `lt` and ends at `from`, -1 when
|
|
118
|
+
* none closes it, or undefined when the pass has read `budget` characters without finding it (the next call goes on).
|
|
119
|
+
* One pass over the page's tags from its first template, with tags of its own (tagsOf), only ever forward: a start
|
|
120
|
+
* tag of a template is opened, an end tag closes the innermost one open, and an element that holds text (TEXT_ONLY:
|
|
121
|
+
* a script, a textarea, a title) is passed to its end tag, its text never read as tags. Each template's end is kept as the pass meets it, so a later
|
|
122
|
+
* template asks the pass again only when it is past where the pass has read. A template the pass did not meet as a
|
|
123
|
+
* tag (one inside a block the two scans read differently) ends at the first end tag of its name, as before.
|
|
124
|
+
*/
|
|
125
|
+
#templateEnd(lt, from, budget) {
|
|
126
|
+
const pass = (this.#templates ??= { next: tagsOf(this.html), at: lt, open: [], seen: new Set(), ends: new Map(), done: false });
|
|
127
|
+
const stop = pass.at + budget;
|
|
128
|
+
while (!pass.ends.has(lt) && !pass.done && (pass.at <= lt || pass.seen.has(lt))) {
|
|
129
|
+
if (pass.at >= stop)
|
|
130
|
+
return undefined;
|
|
131
|
+
const tag = pass.next(pass.at);
|
|
132
|
+
// No tag left, or one with no ">" after it (a comment with no "-->" among them): nothing closes after it.
|
|
133
|
+
if (!tag || tag.gt < 0) {
|
|
134
|
+
pass.done = true;
|
|
135
|
+
break;
|
|
136
|
+
}
|
|
137
|
+
pass.at = tag.gt + 1;
|
|
138
|
+
if (tag.name === "template") {
|
|
139
|
+
if (!tag.end) {
|
|
140
|
+
pass.open.push(tag.lt);
|
|
141
|
+
pass.seen.add(tag.lt);
|
|
142
|
+
}
|
|
143
|
+
else {
|
|
144
|
+
const opened = pass.open.pop();
|
|
145
|
+
if (opened !== undefined)
|
|
146
|
+
pass.ends.set(opened, tag.gt + 1);
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
else if (!tag.end && TEXT_ONLY.has(tag.name)) {
|
|
150
|
+
const close = this.#endTagEnd(tag.name, pass.at);
|
|
151
|
+
if (close < 0)
|
|
152
|
+
pass.done = true;
|
|
153
|
+
else
|
|
154
|
+
pass.at = close;
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
const end = pass.ends.get(lt);
|
|
158
|
+
if (end !== undefined)
|
|
159
|
+
return end;
|
|
160
|
+
return pass.seen.has(lt) ? -1 : this.#endTagEnd("template", from);
|
|
161
|
+
}
|
|
162
|
+
#finish() {
|
|
163
|
+
this.#kept.push(this.html.slice(this.#from));
|
|
164
|
+
this.#from = this.#at = this.html.length;
|
|
165
|
+
this.#done = true;
|
|
166
|
+
}
|
|
167
|
+
/** The page as dropped so far (the whole of it once step has answered true). */
|
|
168
|
+
result() { return this.#kept.length === 1 ? this.#kept[0] : this.#kept.join(""); }
|
|
169
|
+
}
|
|
170
|
+
/** `html` without its inert blocks, all at once (see InertDrop). */
|
|
171
|
+
export function dropInert(html) {
|
|
172
|
+
const drop = new InertDrop(html);
|
|
173
|
+
drop.step(Number.POSITIVE_INFINITY);
|
|
174
|
+
return drop.result();
|
|
175
|
+
}
|
|
176
|
+
/** `html` cut to at most `max` characters, after the last tag that ends before the cut, so no tag is left half; at
|
|
177
|
+
* `max` itself when that tag ends more than 64 KiB before it (a page of one long text), or one before it when `max`
|
|
178
|
+
* falls between the two halves of a character written as a surrogate pair, so no half of one is left. */
|
|
179
|
+
export function cutHtml(html, max) {
|
|
180
|
+
if (html.length <= max)
|
|
181
|
+
return html;
|
|
182
|
+
const gt = html.lastIndexOf(">", max - 1);
|
|
183
|
+
let at = gt >= 0 && max - gt <= 65_536 ? gt + 1 : max;
|
|
184
|
+
if ((html.charCodeAt(at - 1) & 0xfc00) === 0xd800 && (html.charCodeAt(at) & 0xfc00) === 0xdc00)
|
|
185
|
+
at -= 1;
|
|
186
|
+
return html.slice(0, at);
|
|
187
|
+
}
|
|
188
|
+
/** Tags whose start and end each begin a paragraph of the plain text; LINES, a line; CELLS, a cell of a row. */
|
|
189
|
+
const BLOCKS = new Set(["address", "article", "aside", "blockquote", "body", "center", "details", "dialog", "dir", "div", "dl", "fieldset", "figure", "footer", "form", "h1", "h2", "h3", "h4", "h5", "h6", "header", "hgroup", "hr", "main", "menu", "nav", "ol", "p", "pre", "section", "table", "tbody", "tfoot", "thead", "ul"]);
|
|
190
|
+
const LINES = new Set(["br", "caption", "dd", "dt", "figcaption", "legend", "li", "option", "summary", "tr"]);
|
|
191
|
+
const CELLS = new Set(["td", "th"]);
|
|
192
|
+
const CELL = "\u0000";
|
|
193
|
+
const NAME = /[a-zA-Z][a-zA-Z0-9:-]*/y;
|
|
194
|
+
const isSpace = (code) => code === 32 || (code >= 9 && code <= 13);
|
|
195
|
+
const SLASH = 47;
|
|
196
|
+
const EQUALS = 61;
|
|
197
|
+
/** Whether `code` ends an attribute's name: a space, "/", "=" or ">". */
|
|
198
|
+
const endsName = (code) => isSpace(code) || code === SLASH || code === EQUALS || code === 62;
|
|
199
|
+
/**
|
|
200
|
+
* A reader of `html`'s tags, each call the next tag from `at`: "<" then a letter starts one, "</" an end tag, "<!" a
|
|
201
|
+
* declaration or a comment (which ends at its "-->") and "<?" an instruction; any other "<" is text, and passed. Null
|
|
202
|
+
* when none is left. A ">" inside a quoted attribute value (`title="x > y"`) does not end the tag. A quote that no quote
|
|
203
|
+
* closes is no quote: the tag ends at its first ">", so an open quote never takes the rest of the page. Each search
|
|
204
|
+
* remembers what it found, so a scan whose `at` moves forward reads the page once whatever its markup (no "=" for
|
|
205
|
+
* pages, or a thousand open quotes).
|
|
206
|
+
*/
|
|
207
|
+
export function tagsOf(html) {
|
|
208
|
+
const found = new Map();
|
|
209
|
+
/** html.indexOf(`char`, `from`), from what an earlier search found when it still holds. */
|
|
210
|
+
const find = (char, from) => {
|
|
211
|
+
const known = found.get(char);
|
|
212
|
+
if (known && known.from <= from && (known.at < 0 || known.at >= from))
|
|
213
|
+
return known.at;
|
|
214
|
+
const at = html.indexOf(char, from);
|
|
215
|
+
found.set(char, { from, at });
|
|
216
|
+
return at;
|
|
217
|
+
};
|
|
218
|
+
/** The ">" that ends the tag whose name ends at `from`, past quoted values; -1 when there is none. */
|
|
219
|
+
const tagEnd = (from) => {
|
|
220
|
+
for (let at = from;;) {
|
|
221
|
+
const gt = find(">", at);
|
|
222
|
+
if (gt < 0)
|
|
223
|
+
return -1;
|
|
224
|
+
const eq = find("=", at);
|
|
225
|
+
if (eq < 0 || eq > gt)
|
|
226
|
+
return gt;
|
|
227
|
+
let value = eq + 1;
|
|
228
|
+
while (isSpace(html.charCodeAt(value)))
|
|
229
|
+
value += 1;
|
|
230
|
+
const quote = html[value];
|
|
231
|
+
if (quote !== '"' && quote !== "'") {
|
|
232
|
+
at = value;
|
|
233
|
+
continue;
|
|
234
|
+
}
|
|
235
|
+
const close = find(quote, value + 1);
|
|
236
|
+
if (close < 0)
|
|
237
|
+
return gt;
|
|
238
|
+
at = close + 1;
|
|
239
|
+
}
|
|
240
|
+
};
|
|
241
|
+
return (at) => {
|
|
242
|
+
for (let lt = html.indexOf("<", at); lt >= 0; lt = html.indexOf("<", lt + 1)) {
|
|
243
|
+
const next = html.charCodeAt(lt + 1);
|
|
244
|
+
const letter = (next | 32) >= 97 && (next | 32) <= 122;
|
|
245
|
+
const end = next === 47;
|
|
246
|
+
if (!letter && !end && next !== 33 && next !== 63)
|
|
247
|
+
continue;
|
|
248
|
+
if (html.startsWith("!--", lt + 1)) {
|
|
249
|
+
// "<!-->" and "<!--->" are whole comments too.
|
|
250
|
+
const close = html.indexOf("-->", lt + 2);
|
|
251
|
+
return { lt, gt: close < 0 ? -1 : close + 2, name: "", end: false };
|
|
252
|
+
}
|
|
253
|
+
let name = "";
|
|
254
|
+
let from = lt + 2;
|
|
255
|
+
if (letter || end) {
|
|
256
|
+
NAME.lastIndex = end ? lt + 2 : lt + 1;
|
|
257
|
+
const named = NAME.exec(html);
|
|
258
|
+
if (named) {
|
|
259
|
+
name = named[0].toLowerCase();
|
|
260
|
+
from = NAME.lastIndex;
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
return { lt, gt: tagEnd(from), name, end };
|
|
264
|
+
}
|
|
265
|
+
return null;
|
|
266
|
+
};
|
|
267
|
+
}
|
|
268
|
+
const TITLE_END = /<\/title(?=[\s/>])/gi;
|
|
269
|
+
/**
|
|
270
|
+
* The value of a tag's first `name` attribute (href or src), decoded; "" for one written with no value; null when it has
|
|
271
|
+
* none. `inside` is what the tag holds between its name and its ">" (insideOf). Its attributes are read in one pass, as a
|
|
272
|
+
* browser reads them: a name (up to a space, "/", "=" or ">"; a "=" that starts one is part of it), then "=" and a
|
|
273
|
+
* value, quoted or up to a space, or no value. So a "src=" inside another attribute's quoted value is that value's
|
|
274
|
+
* text, and an attribute right after a closing quote (`class="x"src=...`) is one. A value is quoted by the same rule
|
|
275
|
+
* tagsOf ends a tag by: a quote right after the "=" and its spaces; one that no quote closes runs to the tag's end.
|
|
276
|
+
*/
|
|
277
|
+
function attrOf(inside, name, decode) {
|
|
278
|
+
const { length } = inside;
|
|
279
|
+
for (let at = 0; at < length;) {
|
|
280
|
+
const code = inside.charCodeAt(at);
|
|
281
|
+
if (isSpace(code) || code === SLASH) {
|
|
282
|
+
at += 1;
|
|
283
|
+
continue;
|
|
284
|
+
}
|
|
285
|
+
const from = at;
|
|
286
|
+
do
|
|
287
|
+
at += 1;
|
|
288
|
+
while (at < length && !endsName(inside.charCodeAt(at)));
|
|
289
|
+
const named = at - from === name.length && inside.slice(from, at).toLowerCase() === name;
|
|
290
|
+
while (isSpace(inside.charCodeAt(at)))
|
|
291
|
+
at += 1;
|
|
292
|
+
if (inside.charCodeAt(at) !== EQUALS) {
|
|
293
|
+
if (named)
|
|
294
|
+
return "";
|
|
295
|
+
continue;
|
|
296
|
+
}
|
|
297
|
+
at += 1;
|
|
298
|
+
while (isSpace(inside.charCodeAt(at)))
|
|
299
|
+
at += 1;
|
|
300
|
+
const quote = inside[at];
|
|
301
|
+
let start = at;
|
|
302
|
+
let end;
|
|
303
|
+
if (quote === '"' || quote === "'") {
|
|
304
|
+
start = at + 1;
|
|
305
|
+
const close = inside.indexOf(quote, start);
|
|
306
|
+
end = close < 0 ? length : close;
|
|
307
|
+
at = end + 1;
|
|
308
|
+
}
|
|
309
|
+
else {
|
|
310
|
+
while (at < length && !isSpace(inside.charCodeAt(at)))
|
|
311
|
+
at += 1;
|
|
312
|
+
end = at;
|
|
313
|
+
}
|
|
314
|
+
if (named) {
|
|
315
|
+
const value = inside.slice(start, end);
|
|
316
|
+
return value.includes("&") ? decode(value) : value;
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
return null;
|
|
320
|
+
}
|
|
321
|
+
/** What a tag at `lt` named `name` holds between its name and its ">" at `gt`. */
|
|
322
|
+
const insideOf = (html, name, lt, gt) => html.slice(lt + 1 + name.length, gt);
|
|
323
|
+
const MAIN = ["article", "main", "body"];
|
|
324
|
+
/** The first article element of `html`, else its first main, else its body, as HTML from its start tag to its end tag
|
|
325
|
+
* (or the page's end); the whole of `html` when it has none of them. One pass: an element of the same name inside one
|
|
326
|
+
* is counted, so its end tag does not end the one around it. Run on a page its inert blocks are dropped from, so
|
|
327
|
+
* none is picked from inside a script or a comment. */
|
|
328
|
+
export function mainOf(html) {
|
|
329
|
+
const found = new Map();
|
|
330
|
+
const next = tagsOf(html);
|
|
331
|
+
for (let tag = next(0); tag && tag.gt >= 0; tag = next(tag.gt + 1)) {
|
|
332
|
+
if (!MAIN.includes(tag.name))
|
|
333
|
+
continue;
|
|
334
|
+
const element = found.get(tag.name);
|
|
335
|
+
if (!tag.end) {
|
|
336
|
+
if (!element)
|
|
337
|
+
found.set(tag.name, { from: tag.lt, depth: 1 });
|
|
338
|
+
else if (element.to === undefined)
|
|
339
|
+
element.depth += 1;
|
|
340
|
+
}
|
|
341
|
+
else if (element && element.to === undefined && (element.depth -= 1) === 0) {
|
|
342
|
+
element.to = tag.gt + 1;
|
|
343
|
+
if (tag.name === "article")
|
|
344
|
+
break;
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
for (const name of MAIN) {
|
|
348
|
+
const element = found.get(name);
|
|
349
|
+
if (element)
|
|
350
|
+
return html.slice(element.from, element.to ?? html.length);
|
|
351
|
+
}
|
|
352
|
+
return html;
|
|
353
|
+
}
|
|
354
|
+
/** One pass over a page's tags for its frames, in steps so a caller on the event loop can yield between them: each
|
|
355
|
+
* `iframe` element's, and each `frame` element's inside a `frameset` (a `<frame>` outside one is no element, as in a
|
|
356
|
+
* browser), `src` (decoded; null without one), in page order, with the text inside an iframe element (the fallback a
|
|
357
|
+
* browser does not render, where a block page keeps its message: its tags skipped, its entities decoded, read up to
|
|
358
|
+
* `readChars` characters); and the page's first `<base href>`. Inside an iframe element, only its end tag is a tag. */
|
|
359
|
+
export class FrameScan {
|
|
360
|
+
html;
|
|
361
|
+
decode;
|
|
362
|
+
readChars;
|
|
363
|
+
#at = 0;
|
|
364
|
+
#frameset = 0;
|
|
365
|
+
#base = null;
|
|
366
|
+
#open = null;
|
|
367
|
+
#frames = [];
|
|
368
|
+
#next;
|
|
369
|
+
constructor(html, decode, readChars) {
|
|
370
|
+
this.html = html;
|
|
371
|
+
this.decode = decode;
|
|
372
|
+
this.readChars = readChars;
|
|
373
|
+
this.#next = tagsOf(html);
|
|
374
|
+
}
|
|
375
|
+
get done() { return this.#at >= this.html.length; }
|
|
376
|
+
/** Scans the tags that start in the next `chars` characters (see PageScan.step). */
|
|
377
|
+
step(chars) {
|
|
378
|
+
const { html } = this;
|
|
379
|
+
const until = Math.min(html.length, this.#at + chars);
|
|
380
|
+
while (this.#at < until) {
|
|
381
|
+
const tag = this.#next(this.#at);
|
|
382
|
+
if (!tag) {
|
|
383
|
+
this.#words(html.slice(this.#at));
|
|
384
|
+
this.#at = html.length;
|
|
385
|
+
break;
|
|
386
|
+
}
|
|
387
|
+
this.#words(html.slice(this.#at, tag.lt));
|
|
388
|
+
if (tag.gt < 0) {
|
|
389
|
+
this.#at = html.length;
|
|
390
|
+
break;
|
|
391
|
+
}
|
|
392
|
+
this.#at = tag.gt + 1;
|
|
393
|
+
const { name, end, lt, gt } = tag;
|
|
394
|
+
if (this.#open) {
|
|
395
|
+
if (end && name === "iframe")
|
|
396
|
+
this.#close();
|
|
397
|
+
continue;
|
|
398
|
+
}
|
|
399
|
+
if (name === "frameset")
|
|
400
|
+
this.#frameset = Math.max(0, this.#frameset + (end ? -1 : 1));
|
|
401
|
+
else if (end)
|
|
402
|
+
continue;
|
|
403
|
+
else if (name === "base" && this.#base === null)
|
|
404
|
+
this.#base = attrOf(insideOf(html, name, lt, gt), "href", this.decode);
|
|
405
|
+
else if (name === "iframe")
|
|
406
|
+
this.#open = { src: attrOf(insideOf(html, name, lt, gt), "src", this.decode), text: [], read: 0 };
|
|
407
|
+
else if (name === "frame" && this.#frameset > 0)
|
|
408
|
+
this.#frames.push({ src: attrOf(insideOf(html, name, lt, gt), "src", this.decode), text: "" });
|
|
409
|
+
}
|
|
410
|
+
if (this.done && this.#open)
|
|
411
|
+
this.#close();
|
|
412
|
+
return this.done;
|
|
413
|
+
}
|
|
414
|
+
#words(raw) {
|
|
415
|
+
const open = this.#open;
|
|
416
|
+
if (!open || !raw || open.read >= this.readChars)
|
|
417
|
+
return;
|
|
418
|
+
const part = raw.slice(0, this.readChars - open.read);
|
|
419
|
+
open.read += part.length;
|
|
420
|
+
open.text.push(part.includes("&") ? this.decode(part) : part);
|
|
421
|
+
}
|
|
422
|
+
#close() {
|
|
423
|
+
this.#frames.push({ src: this.#open.src, text: this.#open.text.join("") });
|
|
424
|
+
this.#open = null;
|
|
425
|
+
}
|
|
426
|
+
/** The frames scanned, and the first base href (null without one). */
|
|
427
|
+
result() {
|
|
428
|
+
return { frames: [...this.#frames], base: this.#base };
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
/** One pass over a page's tags, in steps so a caller on the event loop can yield between them: its title (the first
|
|
432
|
+
* title element's), the http(s) URLs its `a` elements link to (each resolved against the page's first `<base href>`,
|
|
433
|
+
* or `url`, without its fragment, once, in page order) and, when asked for, its plain text: the text between its tags,
|
|
434
|
+
* entities decoded and whitespace collapsed, a block's start and end a paragraph, a row's start or a `br` a line, and a
|
|
435
|
+
* row's cells joined with " | ". A "<" that starts no tag is text (tagsOf); one with no ">" after it ends the page. */
|
|
436
|
+
export class PageScan {
|
|
437
|
+
html;
|
|
438
|
+
url;
|
|
439
|
+
decode;
|
|
440
|
+
#at = 0;
|
|
441
|
+
#title = null;
|
|
442
|
+
#base = null;
|
|
443
|
+
#hrefs = [];
|
|
444
|
+
#text;
|
|
445
|
+
#next;
|
|
446
|
+
constructor(html, url, decode, withText) {
|
|
447
|
+
this.html = html;
|
|
448
|
+
this.url = url;
|
|
449
|
+
this.decode = decode;
|
|
450
|
+
this.#text = withText ? [] : null;
|
|
451
|
+
this.#next = tagsOf(html);
|
|
452
|
+
}
|
|
453
|
+
/** Whether the whole page is scanned. */
|
|
454
|
+
get done() { return this.#at >= this.html.length; }
|
|
455
|
+
/** Scans the tags that start in the next `chars` characters (a tag that starts in them is read whole). */
|
|
456
|
+
step(chars) {
|
|
457
|
+
const { html } = this;
|
|
458
|
+
const until = Math.min(html.length, this.#at + chars);
|
|
459
|
+
while (this.#at < until) {
|
|
460
|
+
const tag = this.#next(this.#at);
|
|
461
|
+
if (!tag) {
|
|
462
|
+
this.#words(html.slice(this.#at));
|
|
463
|
+
this.#at = html.length;
|
|
464
|
+
break;
|
|
465
|
+
}
|
|
466
|
+
this.#words(html.slice(this.#at, tag.lt));
|
|
467
|
+
if (tag.gt < 0) {
|
|
468
|
+
this.#at = html.length;
|
|
469
|
+
break;
|
|
470
|
+
}
|
|
471
|
+
this.#at = tag.gt + 1;
|
|
472
|
+
if (tag.name)
|
|
473
|
+
this.#tag(tag.name, tag.end, tag.lt, tag.gt);
|
|
474
|
+
}
|
|
475
|
+
return this.done;
|
|
476
|
+
}
|
|
477
|
+
#tag(name, end, lt, gt) {
|
|
478
|
+
const text = this.#text;
|
|
479
|
+
if (text) {
|
|
480
|
+
if (BLOCKS.has(name))
|
|
481
|
+
text.push("\n\n");
|
|
482
|
+
else if (LINES.has(name))
|
|
483
|
+
text.push("\n");
|
|
484
|
+
else if (CELLS.has(name) && !end)
|
|
485
|
+
text.push(CELL);
|
|
486
|
+
}
|
|
487
|
+
if (end)
|
|
488
|
+
return;
|
|
489
|
+
if (name === "a" || (name === "base" && this.#base === null)) {
|
|
490
|
+
const href = attrOf(insideOf(this.html, name, lt, gt), "href", this.decode);
|
|
491
|
+
if (href === null)
|
|
492
|
+
return;
|
|
493
|
+
if (name === "a")
|
|
494
|
+
this.#hrefs.push(href);
|
|
495
|
+
else
|
|
496
|
+
this.#base = href;
|
|
497
|
+
}
|
|
498
|
+
else if (name === "title" && this.#title === null) {
|
|
499
|
+
// The first title's text is the page's title, never its text: it is read up to its end tag, and the scan goes on
|
|
500
|
+
// there. Only the first is looked for (once), so a page of unclosed titles is not searched to its end at each.
|
|
501
|
+
TITLE_END.lastIndex = gt + 1;
|
|
502
|
+
const close = TITLE_END.exec(this.html);
|
|
503
|
+
const stop = close ? close.index : this.html.indexOf("<", gt + 1);
|
|
504
|
+
this.#title = this.decode(this.html.slice(gt + 1, stop < 0 ? this.html.length : stop)).trim();
|
|
505
|
+
if (close)
|
|
506
|
+
this.#at = Math.max(this.#at, close.index);
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
#words(raw) {
|
|
510
|
+
if (!this.#text || !raw)
|
|
511
|
+
return;
|
|
512
|
+
// A NUL is a space here: it would read as CELL.
|
|
513
|
+
this.#text.push((raw.includes("&") ? this.decode(raw) : raw).replace(/[\s\u0000]+/g, " "));
|
|
514
|
+
}
|
|
515
|
+
/** The title, the links and the plain text ("" when it was not asked for) of what was scanned. */
|
|
516
|
+
result() {
|
|
517
|
+
let base = this.url;
|
|
518
|
+
if (this.#base !== null) {
|
|
519
|
+
try {
|
|
520
|
+
base = new URL(this.#base, this.url).href;
|
|
521
|
+
}
|
|
522
|
+
catch { /* a base that does not parse: the page's own URL stands */ }
|
|
523
|
+
}
|
|
524
|
+
const links = new Set();
|
|
525
|
+
for (const href of this.#hrefs) {
|
|
526
|
+
const link = URL.parse(href, base);
|
|
527
|
+
if (!link || !/^https?:$/.test(link.protocol))
|
|
528
|
+
continue;
|
|
529
|
+
link.hash = "";
|
|
530
|
+
links.add(link.href);
|
|
531
|
+
}
|
|
532
|
+
const text = this.#text ? this.#text.join("")
|
|
533
|
+
.replace(/ {2,}/g, " ")
|
|
534
|
+
.replace(/(^|\n) ?\u0000/g, "$1")
|
|
535
|
+
.replace(/ ?\u0000 ?/g, " | ")
|
|
536
|
+
.replace(/ {2,}/g, " ")
|
|
537
|
+
.replace(/ ?\n ?/g, "\n")
|
|
538
|
+
.replace(/\n{3,}/g, "\n\n")
|
|
539
|
+
.trim() : "";
|
|
540
|
+
return { title: this.#title ?? "", links: [...links], text };
|
|
541
|
+
}
|
|
542
|
+
}
|
|
@@ -30,10 +30,3 @@ export type PublicAnswer = {
|
|
|
30
30
|
* with the signal's reason), and an aborted hop is never dialled.
|
|
31
31
|
*/
|
|
32
32
|
export declare function publicFetch(url: string, options?: PublicFetchOptions): Promise<PublicAnswer>;
|
|
33
|
-
/** An HTML page's title, its text as markdown, and the absolute http(s) URLs it links to (resolved against its
|
|
34
|
-
* `<base href>` or `url`), each once, in page order. */
|
|
35
|
-
export declare function readPage(html: string, url: string): Promise<{
|
|
36
|
-
title: string;
|
|
37
|
-
text: string;
|
|
38
|
-
links: string[];
|
|
39
|
-
}>;
|
|
@@ -132,28 +132,3 @@ export async function publicFetch(url, options = {}) {
|
|
|
132
132
|
}
|
|
133
133
|
}
|
|
134
134
|
}
|
|
135
|
-
/** An HTML page's title, its text as markdown, and the absolute http(s) URLs it links to (resolved against its
|
|
136
|
-
* `<base href>` or `url`), each once, in page order. */
|
|
137
|
-
export async function readPage(html, url) {
|
|
138
|
-
const [{ parse }, { NodeHtmlMarkdown }] = await Promise.all([import("node-html-parser"), import("node-html-markdown")]);
|
|
139
|
-
const root = parse(html);
|
|
140
|
-
let base = url;
|
|
141
|
-
const baseHref = root.querySelector("base[href]")?.getAttribute("href");
|
|
142
|
-
if (baseHref) {
|
|
143
|
-
try {
|
|
144
|
-
base = new URL(baseHref, url).href;
|
|
145
|
-
}
|
|
146
|
-
catch { /* a base that does not parse: the page's own URL stands */ }
|
|
147
|
-
}
|
|
148
|
-
const links = new Set();
|
|
149
|
-
for (const anchor of root.querySelectorAll("a[href]")) {
|
|
150
|
-
try {
|
|
151
|
-
const link = new URL(anchor.getAttribute("href"), base);
|
|
152
|
-
link.hash = "";
|
|
153
|
-
if (/^https?:$/.test(link.protocol))
|
|
154
|
-
links.add(link.href);
|
|
155
|
-
}
|
|
156
|
-
catch { /* an href that does not parse is no link */ }
|
|
157
|
-
}
|
|
158
|
-
return { title: root.querySelector("title")?.text.trim() ?? "", text: NodeHtmlMarkdown.translate(html, { keepDataImages: false }).trim(), links: [...links] };
|
|
159
|
-
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/** `html` as given (readPage drops and cuts it first), or, with `max`, dropped (its first article, main or body only,
|
|
2
|
+
* with `main`) and cut at `max` here; `url`: the page's title and links are read too. */
|
|
3
|
+
export type Job = {
|
|
4
|
+
html: string;
|
|
5
|
+
url?: string;
|
|
6
|
+
main?: boolean;
|
|
7
|
+
max?: number;
|
|
8
|
+
};
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
// read-page.ts's worker. For `workerData`, a Job: the page's markdown, posted once as { text } (with { title, links }
|
|
2
|
+
// when the job has a `url`, and { truncated, html_chars } when the worker cut the page), or as { error } when the
|
|
3
|
+
// conversion threw (a page nested past the stack, say).
|
|
4
|
+
import { parentPort, workerData } from "node:worker_threads";
|
|
5
|
+
import { TextNode } from "node-html-parser";
|
|
6
|
+
import { markdownOf } from "./page-markdown.js";
|
|
7
|
+
import { cutHtml, dropInert, mainOf, PageScan } from "./page-scan.js";
|
|
8
|
+
if (parentPort) {
|
|
9
|
+
const job = workerData;
|
|
10
|
+
try {
|
|
11
|
+
let html = job.html;
|
|
12
|
+
let cut = {};
|
|
13
|
+
if (job.max !== undefined) {
|
|
14
|
+
const kept = dropInert(job.html);
|
|
15
|
+
const picked = job.main ? mainOf(kept) : kept;
|
|
16
|
+
html = cutHtml(picked, job.max);
|
|
17
|
+
if (html.length < picked.length)
|
|
18
|
+
cut = { truncated: true, html_chars: job.html.length };
|
|
19
|
+
}
|
|
20
|
+
let read = {};
|
|
21
|
+
if (job.url !== undefined) {
|
|
22
|
+
const scan = new PageScan(html, job.url, (text) => new TextNode(text).text, false);
|
|
23
|
+
scan.step(Number.POSITIVE_INFINITY);
|
|
24
|
+
const { title, links } = scan.result();
|
|
25
|
+
read = { title, links };
|
|
26
|
+
}
|
|
27
|
+
parentPort.postMessage({ ...read, ...cut, text: markdownOf(html) });
|
|
28
|
+
}
|
|
29
|
+
catch (error) {
|
|
30
|
+
parentPort.postMessage({ error: String(error?.message ?? error) });
|
|
31
|
+
}
|
|
32
|
+
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/** The most of a page's HTML that is read, counted once its script, style, svg, noscript and template blocks and its
|
|
2
|
+
* comments are dropped. A million characters of go.dev/dl converts in about 0.8 s on a busy 48-core host and gives
|
|
3
|
+
* 590,000 characters of markdown, ten times the 60,000 one web_fetch answer carries. */
|
|
4
|
+
export declare const READ_MAX_CHARS = 1000000;
|
|
5
|
+
/** How long the worker has for the markdown by default: six times what READ_MAX_CHARS of tables took. */
|
|
6
|
+
export declare const READ_DEADLINE_MS = 5000;
|
|
7
|
+
/** `n` with its digits in groups of three ("3,154,060"): Intl's first use costs tens of milliseconds on the loop. */
|
|
8
|
+
export declare const thousands: (n: number) => string;
|
|
9
|
+
/** `converted`: "markdown", or "text" when the conversion did not finish in time (or failed) and the text is the page's
|
|
10
|
+
* plain text. `truncated` and `html_chars` (the page's whole length, in characters) when the page was cut at
|
|
11
|
+
* READ_MAX_CHARS before it was read: the text, the title and the links are then the cut page's. */
|
|
12
|
+
export type PageRead = {
|
|
13
|
+
title: string;
|
|
14
|
+
text: string;
|
|
15
|
+
links: string[];
|
|
16
|
+
converted: "markdown" | "text";
|
|
17
|
+
truncated?: true;
|
|
18
|
+
html_chars?: number;
|
|
19
|
+
};
|
|
20
|
+
/** `deadlineMs`: how long the conversion to markdown has (default READ_DEADLINE_MS). `signal` ends the read, the
|
|
21
|
+
* conversion stopped, with the signal's reason. */
|
|
22
|
+
export type ReadPageOptions = {
|
|
23
|
+
deadlineMs?: number;
|
|
24
|
+
signal?: AbortSignal;
|
|
25
|
+
};
|
|
26
|
+
/**
|
|
27
|
+
* An HTML page's title, its text as markdown, and the absolute http(s) URLs it links to (resolved against its
|
|
28
|
+
* `<base href>` or `url`), each once, in page order. The event loop is never held for longer than one linear step of a
|
|
29
|
+
* bounded part of the page (see the file's head and PageRead). Rejects only with `signal`'s reason.
|
|
30
|
+
*/
|
|
31
|
+
export declare function readPage(html: string, url: string, { deadlineMs, signal }?: ReadPageOptions): Promise<PageRead>;
|
|
32
|
+
/**
|
|
33
|
+
* `html`'s markdown, made in the worker within `deadlineMs` (default READ_DEADLINE_MS): of its first article, else its
|
|
34
|
+
* main, else its body element when `main` (mainOf), once its inert blocks are dropped, cut at READ_MAX_CHARS (then
|
|
35
|
+
* `truncated`, and `html_chars` is the page's whole length). Null when the worker has not answered in time, failed, or
|
|
36
|
+
* could not start. The event loop does no more than hand the worker the page. Rejects only with `signal`'s reason.
|
|
37
|
+
*/
|
|
38
|
+
export declare function htmlMarkdown(html: string, { main, deadlineMs, signal }?: {
|
|
39
|
+
main?: boolean;
|
|
40
|
+
} & ReadPageOptions): Promise<{
|
|
41
|
+
text: string;
|
|
42
|
+
truncated?: true;
|
|
43
|
+
html_chars?: number;
|
|
44
|
+
} | null>;
|
|
45
|
+
/** `html` without its inert blocks (InertDrop), STEP_CHARS at a time with the loop yielded between steps. */
|
|
46
|
+
export declare function dropOnLoop(html: string, signal?: AbortSignal): Promise<string>;
|