@parcha/pi-browser 0.1.0-beta.28 → 0.1.0-beta.29

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,95 @@
1
+ /** An entity decoder for a run of text or an attribute value: node-html-parser's in the worker, entities.ts's on the
2
+ * event loop. */
3
+ export type Decode = (text: string) => string;
4
+ /** A page without its script, style, svg, noscript and template elements and its comments, in steps so a caller on the
5
+ * event loop can yield between them (dropInert, all at once). One whose end is missing runs to the end of the page, as
6
+ * a browser reads it; but an svg's start tag goes alone, since a browser lets the HTML that follows out of an unclosed
7
+ * svg. A template ends at the end tag that closes it, as in a browser: a template inside it is counted, so its end tag
8
+ * ends only the inner one, and the text of an element that holds text, not markup, is never read as tags in it
9
+ * (TEXT_ONLY: a script, a textarea, a title). Out of a template, a textarea's or a title's text is not read as tags
10
+ * either: a "<template>" or a "<script>" in it is text, kept with each "<" written "&lt;" (KEPT_TEXT). A nested svg ends at the first end tag of its name. A search for an end tag that finds none is not made again from
11
+ * a later start of that name (none follows it either), so a page of unclosed svgs is read once, not once for each; and
12
+ * the templates' ends are found in one forward pass however many there are (#templateEnd). */
13
+ export declare class InertDrop {
14
+ #private;
15
+ readonly html: string;
16
+ constructor(html: string);
17
+ /** Drops the blocks found in the next `chars` characters or so; true once the whole page is read. A template's end
18
+ * is looked for `chars` characters at a time too, so no step reads much more than that. */
19
+ step(chars: number): boolean;
20
+ /** The page as dropped so far (the whole of it once step has answered true). */
21
+ result(): string;
22
+ }
23
+ /** `html` without its inert blocks, all at once (see InertDrop). */
24
+ export declare function dropInert(html: string): string;
25
+ /** `html` cut to at most `max` characters, after the last tag that ends before the cut, so no tag is left half; at
26
+ * `max` itself when that tag ends more than 64 KiB before it (a page of one long text), or one before it when `max`
27
+ * falls between the two halves of a character written as a surrogate pair, so no half of one is left. */
28
+ export declare function cutHtml(html: string, max: number): string;
29
+ /** A tag in a page: `lt` and `gt` the offsets of its "<" and ">", `name` lowercase ("" for a declaration, a comment
30
+ * or an instruction), `end` for an end tag; `gt` is -1 for a "<" that starts a tag with no ">" after it. */
31
+ export type Tag = {
32
+ lt: number;
33
+ gt: number;
34
+ name: string;
35
+ end: boolean;
36
+ };
37
+ /**
38
+ * A reader of `html`'s tags, each call the next tag from `at`: "<" then a letter starts one, "</" an end tag, "<!" a
39
+ * declaration or a comment (which ends at its "-->") and "<?" an instruction; any other "<" is text, and passed. Null
40
+ * when none is left. A ">" inside a quoted attribute value (`title="x > y"`) does not end the tag. A quote that no quote
41
+ * closes is no quote: the tag ends at its first ">", so an open quote never takes the rest of the page. Each search
42
+ * remembers what it found, so a scan whose `at` moves forward reads the page once whatever its markup (no "=" for
43
+ * pages, or a thousand open quotes).
44
+ */
45
+ export declare function tagsOf(html: string): (at: number) => Tag | null;
46
+ /** The first article element of `html`, else its first main, else its body, as HTML from its start tag to its end tag
47
+ * (or the page's end); the whole of `html` when it has none of them. One pass: an element of the same name inside one
48
+ * is counted, so its end tag does not end the one around it. Run on a page its inert blocks are dropped from, so
49
+ * none is picked from inside a script or a comment. */
50
+ export declare function mainOf(html: string): string;
51
+ /** One pass over a page's tags for its frames, in steps so a caller on the event loop can yield between them: each
52
+ * `iframe` element's, and each `frame` element's inside a `frameset` (a `<frame>` outside one is no element, as in a
53
+ * browser), `src` (decoded; null without one), in page order, with the text inside an iframe element (the fallback a
54
+ * browser does not render, where a block page keeps its message: its tags skipped, its entities decoded, read up to
55
+ * `readChars` characters); and the page's first `<base href>`. Inside an iframe element, only its end tag is a tag. */
56
+ export declare class FrameScan {
57
+ #private;
58
+ readonly html: string;
59
+ readonly decode: Decode;
60
+ readonly readChars: number;
61
+ constructor(html: string, decode: Decode, readChars: number);
62
+ get done(): boolean;
63
+ /** Scans the tags that start in the next `chars` characters (see PageScan.step). */
64
+ step(chars: number): boolean;
65
+ /** The frames scanned, and the first base href (null without one). */
66
+ result(): {
67
+ frames: Array<{
68
+ src: string | null;
69
+ text: string;
70
+ }>;
71
+ base: string | null;
72
+ };
73
+ }
74
+ /** One pass over a page's tags, in steps so a caller on the event loop can yield between them: its title (the first
75
+ * title element's), the http(s) URLs its `a` elements link to (each resolved against the page's first `<base href>`,
76
+ * or `url`, without its fragment, once, in page order) and, when asked for, its plain text: the text between its tags,
77
+ * entities decoded and whitespace collapsed, a block's start and end a paragraph, a row's start or a `br` a line, and a
78
+ * row's cells joined with " | ". A "<" that starts no tag is text (tagsOf); one with no ">" after it ends the page. */
79
+ export declare class PageScan {
80
+ #private;
81
+ readonly html: string;
82
+ readonly url: string;
83
+ readonly decode: Decode;
84
+ constructor(html: string, url: string, decode: Decode, withText: boolean);
85
+ /** Whether the whole page is scanned. */
86
+ get done(): boolean;
87
+ /** Scans the tags that start in the next `chars` characters (a tag that starts in them is read whole). */
88
+ step(chars: number): boolean;
89
+ /** The title, the links and the plain text ("" when it was not asked for) of what was scanned. */
90
+ result(): {
91
+ title: string;
92
+ links: string[];
93
+ text: string;
94
+ };
95
+ }
@@ -0,0 +1,542 @@
1
+ // The readings of an HTML page that need no DOM: the page without the blocks that hold no readable text, the page cut
2
+ // to a length, and one pass over its tags for its title, its links and its plain text. Each is linear in the page's
3
+ // length whatever its markup (a page with no ">" after a "<", a thousand unclosed scripts): a scan with indexOf, or a
4
+ // pattern whose every try is bounded by the next character it can stop at. read-page.ts runs them on the event loop;
5
+ // its worker runs the scan for the title and links.
6
+ const INERT_NAMES = new Set(["script", "style", "svg", "noscript", "template"]);
7
+ /** The elements whose content is text, not markup, to their first end tag, as a browser reads them: a tag inside one is
8
+ * none (script, style and noscript are also inert). */
9
+ const TEXT_ONLY = new Set(["script", "style", "noscript", "textarea", "title", "xmp", "iframe", "noembed", "noframes"]);
10
+ /** The kept ones whose text the drop also writes as text: each "<" in a textarea's or a title's text becomes "&lt;", so
11
+ * a parser that reads them as markup (node-html-parser does) reads the text a browser shows. */
12
+ const KEPT_TEXT = new Set(["textarea", "title"]);
13
+ const CLOSER = Object.fromEntries([...TEXT_ONLY, "svg", "template"].map((name) => [name, new RegExp(`</${name}(?=[\\s/>])[^>]*>?`, "gi")]));
14
+ /** A page without its script, style, svg, noscript and template elements and its comments, in steps so a caller on the
15
+ * event loop can yield between them (dropInert, all at once). One whose end is missing runs to the end of the page, as
16
+ * a browser reads it; but an svg's start tag goes alone, since a browser lets the HTML that follows out of an unclosed
17
+ * svg. A template ends at the end tag that closes it, as in a browser: a template inside it is counted, so its end tag
18
+ * ends only the inner one, and the text of an element that holds text, not markup, is never read as tags in it
19
+ * (TEXT_ONLY: a script, a textarea, a title). Out of a template, a textarea's or a title's text is not read as tags
20
+ * either: a "<template>" or a "<script>" in it is text, kept with each "<" written "&lt;" (KEPT_TEXT). A nested svg ends at the first end tag of its name. A search for an end tag that finds none is not made again from
21
+ * a later start of that name (none follows it either), so a page of unclosed svgs is read once, not once for each; and
22
+ * the templates' ends are found in one forward pass however many there are (#templateEnd). */
23
+ export class InertDrop {
24
+ html;
25
+ // Its blocks are found among the page's tags (tagsOf), so a "<script>" inside an attribute value is none.
26
+ #next;
27
+ #unclosed = new Map();
28
+ #kept = [];
29
+ /** Where the scan is, and where the HTML kept since the last block dropped starts. */
30
+ #at = 0;
31
+ #from = 0;
32
+ #done = false;
33
+ #templates = null;
34
+ constructor(html) {
35
+ this.html = html;
36
+ this.#next = tagsOf(html);
37
+ }
38
+ /** Drops the blocks found in the next `chars` characters or so; true once the whole page is read. A template's end
39
+ * is looked for `chars` characters at a time too, so no step reads much more than that. */
40
+ step(chars) {
41
+ const { html } = this;
42
+ const until = this.#at + chars;
43
+ while (!this.#done && this.#at < until) {
44
+ const tag = this.#next(this.#at);
45
+ if (!tag) {
46
+ this.#finish();
47
+ break;
48
+ }
49
+ const comment = html.startsWith("!--", tag.lt + 1);
50
+ // A tag with no ">" after it: the rest of the page is kept as it is (a comment with no "-->" runs to its end).
51
+ if (!comment && tag.gt < 0) {
52
+ this.#finish();
53
+ break;
54
+ }
55
+ if (!comment && (tag.end || !INERT_NAMES.has(tag.name))) {
56
+ if (tag.end || !KEPT_TEXT.has(tag.name)) {
57
+ this.#at = tag.gt + 1;
58
+ continue;
59
+ }
60
+ // A textarea or a title: its text is passed to its end tag, never read as tags, so a "<script>" or a
61
+ // "<template>" in it drops nothing, and is kept with each "<" written "&lt;"; with no end tag the rest of the
62
+ // page is its text.
63
+ const close = this.#endTag(tag.name, tag.gt + 1);
64
+ const text = html.slice(tag.gt + 1, close ? close.start : html.length);
65
+ if (text.includes("<")) {
66
+ this.#kept.push(html.slice(this.#from, tag.gt + 1), text.replaceAll("<", "&lt;"));
67
+ this.#from = close ? close.start : html.length;
68
+ }
69
+ if (!close) {
70
+ this.#finish();
71
+ break;
72
+ }
73
+ this.#at = close.end;
74
+ continue;
75
+ }
76
+ let end = tag.gt < 0 ? html.length : tag.gt + 1;
77
+ if (!comment) {
78
+ const name = tag.name;
79
+ if (name === "template") {
80
+ const close = this.#templateEnd(tag.lt, end, chars);
81
+ // Not found within this step's reading: the next step goes on from this template.
82
+ if (close === undefined)
83
+ break;
84
+ end = close >= 0 ? close : html.length;
85
+ }
86
+ else {
87
+ const close = this.#endTagEnd(name, end);
88
+ end = close >= 0 ? close : name === "svg" ? end : html.length;
89
+ }
90
+ }
91
+ this.#kept.push(html.slice(this.#from, tag.lt));
92
+ this.#from = this.#at = end;
93
+ if (end >= html.length)
94
+ this.#finish();
95
+ }
96
+ return this.#done;
97
+ }
98
+ /** The first end tag named `name` from `from` (CLOSER), where it starts and the offset just past it; null when there
99
+ * is none. */
100
+ #endTag(name, from) {
101
+ if (from >= (this.#unclosed.get(name) ?? Number.POSITIVE_INFINITY))
102
+ return null;
103
+ const closer = CLOSER[name];
104
+ closer.lastIndex = from;
105
+ const close = closer.exec(this.html);
106
+ if (!close) {
107
+ this.#unclosed.set(name, from);
108
+ return null;
109
+ }
110
+ return { start: close.index, end: close.index + close[0].length };
111
+ }
112
+ /** The offset just past the first end tag named `name` from `from`, or -1 when there is none (#endTag). */
113
+ #endTagEnd(name, from) {
114
+ return this.#endTag(name, from)?.end ?? -1;
115
+ }
116
+ /**
117
+ * The offset just past the end tag that closes the template whose start tag is at `lt` and ends at `from`, -1 when
118
+ * none closes it, or undefined when the pass has read `budget` characters without finding it (the next call goes on).
119
+ * One pass over the page's tags from its first template, with tags of its own (tagsOf), only ever forward: a start
120
+ * tag of a template is opened, an end tag closes the innermost one open, and an element that holds text (TEXT_ONLY:
121
+ * a script, a textarea, a title) is passed to its end tag, its text never read as tags. Each template's end is kept as the pass meets it, so a later
122
+ * template asks the pass again only when it is past where the pass has read. A template the pass did not meet as a
123
+ * tag (one inside a block the two scans read differently) ends at the first end tag of its name, as before.
124
+ */
125
+ #templateEnd(lt, from, budget) {
126
+ const pass = (this.#templates ??= { next: tagsOf(this.html), at: lt, open: [], seen: new Set(), ends: new Map(), done: false });
127
+ const stop = pass.at + budget;
128
+ while (!pass.ends.has(lt) && !pass.done && (pass.at <= lt || pass.seen.has(lt))) {
129
+ if (pass.at >= stop)
130
+ return undefined;
131
+ const tag = pass.next(pass.at);
132
+ // No tag left, or one with no ">" after it (a comment with no "-->" among them): nothing closes after it.
133
+ if (!tag || tag.gt < 0) {
134
+ pass.done = true;
135
+ break;
136
+ }
137
+ pass.at = tag.gt + 1;
138
+ if (tag.name === "template") {
139
+ if (!tag.end) {
140
+ pass.open.push(tag.lt);
141
+ pass.seen.add(tag.lt);
142
+ }
143
+ else {
144
+ const opened = pass.open.pop();
145
+ if (opened !== undefined)
146
+ pass.ends.set(opened, tag.gt + 1);
147
+ }
148
+ }
149
+ else if (!tag.end && TEXT_ONLY.has(tag.name)) {
150
+ const close = this.#endTagEnd(tag.name, pass.at);
151
+ if (close < 0)
152
+ pass.done = true;
153
+ else
154
+ pass.at = close;
155
+ }
156
+ }
157
+ const end = pass.ends.get(lt);
158
+ if (end !== undefined)
159
+ return end;
160
+ return pass.seen.has(lt) ? -1 : this.#endTagEnd("template", from);
161
+ }
162
+ #finish() {
163
+ this.#kept.push(this.html.slice(this.#from));
164
+ this.#from = this.#at = this.html.length;
165
+ this.#done = true;
166
+ }
167
+ /** The page as dropped so far (the whole of it once step has answered true). */
168
+ result() { return this.#kept.length === 1 ? this.#kept[0] : this.#kept.join(""); }
169
+ }
170
+ /** `html` without its inert blocks, all at once (see InertDrop). */
171
+ export function dropInert(html) {
172
+ const drop = new InertDrop(html);
173
+ drop.step(Number.POSITIVE_INFINITY);
174
+ return drop.result();
175
+ }
176
+ /** `html` cut to at most `max` characters, after the last tag that ends before the cut, so no tag is left half; at
177
+ * `max` itself when that tag ends more than 64 KiB before it (a page of one long text), or one before it when `max`
178
+ * falls between the two halves of a character written as a surrogate pair, so no half of one is left. */
179
+ export function cutHtml(html, max) {
180
+ if (html.length <= max)
181
+ return html;
182
+ const gt = html.lastIndexOf(">", max - 1);
183
+ let at = gt >= 0 && max - gt <= 65_536 ? gt + 1 : max;
184
+ if ((html.charCodeAt(at - 1) & 0xfc00) === 0xd800 && (html.charCodeAt(at) & 0xfc00) === 0xdc00)
185
+ at -= 1;
186
+ return html.slice(0, at);
187
+ }
188
+ /** Tags whose start and end each begin a paragraph of the plain text; LINES, a line; CELLS, a cell of a row. */
189
+ const BLOCKS = new Set(["address", "article", "aside", "blockquote", "body", "center", "details", "dialog", "dir", "div", "dl", "fieldset", "figure", "footer", "form", "h1", "h2", "h3", "h4", "h5", "h6", "header", "hgroup", "hr", "main", "menu", "nav", "ol", "p", "pre", "section", "table", "tbody", "tfoot", "thead", "ul"]);
190
+ const LINES = new Set(["br", "caption", "dd", "dt", "figcaption", "legend", "li", "option", "summary", "tr"]);
191
+ const CELLS = new Set(["td", "th"]);
192
+ const CELL = "\u0000";
193
+ const NAME = /[a-zA-Z][a-zA-Z0-9:-]*/y;
194
+ const isSpace = (code) => code === 32 || (code >= 9 && code <= 13);
195
+ const SLASH = 47;
196
+ const EQUALS = 61;
197
+ /** Whether `code` ends an attribute's name: a space, "/", "=" or ">". */
198
+ const endsName = (code) => isSpace(code) || code === SLASH || code === EQUALS || code === 62;
199
+ /**
200
+ * A reader of `html`'s tags, each call the next tag from `at`: "<" then a letter starts one, "</" an end tag, "<!" a
201
+ * declaration or a comment (which ends at its "-->") and "<?" an instruction; any other "<" is text, and passed. Null
202
+ * when none is left. A ">" inside a quoted attribute value (`title="x > y"`) does not end the tag. A quote that no quote
203
+ * closes is no quote: the tag ends at its first ">", so an open quote never takes the rest of the page. Each search
204
+ * remembers what it found, so a scan whose `at` moves forward reads the page once whatever its markup (no "=" for
205
+ * pages, or a thousand open quotes).
206
+ */
207
+ export function tagsOf(html) {
208
+ const found = new Map();
209
+ /** html.indexOf(`char`, `from`), from what an earlier search found when it still holds. */
210
+ const find = (char, from) => {
211
+ const known = found.get(char);
212
+ if (known && known.from <= from && (known.at < 0 || known.at >= from))
213
+ return known.at;
214
+ const at = html.indexOf(char, from);
215
+ found.set(char, { from, at });
216
+ return at;
217
+ };
218
+ /** The ">" that ends the tag whose name ends at `from`, past quoted values; -1 when there is none. */
219
+ const tagEnd = (from) => {
220
+ for (let at = from;;) {
221
+ const gt = find(">", at);
222
+ if (gt < 0)
223
+ return -1;
224
+ const eq = find("=", at);
225
+ if (eq < 0 || eq > gt)
226
+ return gt;
227
+ let value = eq + 1;
228
+ while (isSpace(html.charCodeAt(value)))
229
+ value += 1;
230
+ const quote = html[value];
231
+ if (quote !== '"' && quote !== "'") {
232
+ at = value;
233
+ continue;
234
+ }
235
+ const close = find(quote, value + 1);
236
+ if (close < 0)
237
+ return gt;
238
+ at = close + 1;
239
+ }
240
+ };
241
+ return (at) => {
242
+ for (let lt = html.indexOf("<", at); lt >= 0; lt = html.indexOf("<", lt + 1)) {
243
+ const next = html.charCodeAt(lt + 1);
244
+ const letter = (next | 32) >= 97 && (next | 32) <= 122;
245
+ const end = next === 47;
246
+ if (!letter && !end && next !== 33 && next !== 63)
247
+ continue;
248
+ if (html.startsWith("!--", lt + 1)) {
249
+ // "<!-->" and "<!--->" are whole comments too.
250
+ const close = html.indexOf("-->", lt + 2);
251
+ return { lt, gt: close < 0 ? -1 : close + 2, name: "", end: false };
252
+ }
253
+ let name = "";
254
+ let from = lt + 2;
255
+ if (letter || end) {
256
+ NAME.lastIndex = end ? lt + 2 : lt + 1;
257
+ const named = NAME.exec(html);
258
+ if (named) {
259
+ name = named[0].toLowerCase();
260
+ from = NAME.lastIndex;
261
+ }
262
+ }
263
+ return { lt, gt: tagEnd(from), name, end };
264
+ }
265
+ return null;
266
+ };
267
+ }
268
+ const TITLE_END = /<\/title(?=[\s/>])/gi;
269
+ /**
270
+ * The value of a tag's first `name` attribute (href or src), decoded; "" for one written with no value; null when it has
271
+ * none. `inside` is what the tag holds between its name and its ">" (insideOf). Its attributes are read in one pass, as a
272
+ * browser reads them: a name (up to a space, "/", "=" or ">"; a "=" that starts one is part of it), then "=" and a
273
+ * value, quoted or up to a space, or no value. So a "src=" inside another attribute's quoted value is that value's
274
+ * text, and an attribute right after a closing quote (`class="x"src=...`) is one. A value is quoted by the same rule
275
+ * tagsOf ends a tag by: a quote right after the "=" and its spaces; one that no quote closes runs to the tag's end.
276
+ */
277
+ function attrOf(inside, name, decode) {
278
+ const { length } = inside;
279
+ for (let at = 0; at < length;) {
280
+ const code = inside.charCodeAt(at);
281
+ if (isSpace(code) || code === SLASH) {
282
+ at += 1;
283
+ continue;
284
+ }
285
+ const from = at;
286
+ do
287
+ at += 1;
288
+ while (at < length && !endsName(inside.charCodeAt(at)));
289
+ const named = at - from === name.length && inside.slice(from, at).toLowerCase() === name;
290
+ while (isSpace(inside.charCodeAt(at)))
291
+ at += 1;
292
+ if (inside.charCodeAt(at) !== EQUALS) {
293
+ if (named)
294
+ return "";
295
+ continue;
296
+ }
297
+ at += 1;
298
+ while (isSpace(inside.charCodeAt(at)))
299
+ at += 1;
300
+ const quote = inside[at];
301
+ let start = at;
302
+ let end;
303
+ if (quote === '"' || quote === "'") {
304
+ start = at + 1;
305
+ const close = inside.indexOf(quote, start);
306
+ end = close < 0 ? length : close;
307
+ at = end + 1;
308
+ }
309
+ else {
310
+ while (at < length && !isSpace(inside.charCodeAt(at)))
311
+ at += 1;
312
+ end = at;
313
+ }
314
+ if (named) {
315
+ const value = inside.slice(start, end);
316
+ return value.includes("&") ? decode(value) : value;
317
+ }
318
+ }
319
+ return null;
320
+ }
321
+ /** What a tag at `lt` named `name` holds between its name and its ">" at `gt`. */
322
+ const insideOf = (html, name, lt, gt) => html.slice(lt + 1 + name.length, gt);
323
+ const MAIN = ["article", "main", "body"];
324
+ /** The first article element of `html`, else its first main, else its body, as HTML from its start tag to its end tag
325
+ * (or the page's end); the whole of `html` when it has none of them. One pass: an element of the same name inside one
326
+ * is counted, so its end tag does not end the one around it. Run on a page its inert blocks are dropped from, so
327
+ * none is picked from inside a script or a comment. */
328
+ export function mainOf(html) {
329
+ const found = new Map();
330
+ const next = tagsOf(html);
331
+ for (let tag = next(0); tag && tag.gt >= 0; tag = next(tag.gt + 1)) {
332
+ if (!MAIN.includes(tag.name))
333
+ continue;
334
+ const element = found.get(tag.name);
335
+ if (!tag.end) {
336
+ if (!element)
337
+ found.set(tag.name, { from: tag.lt, depth: 1 });
338
+ else if (element.to === undefined)
339
+ element.depth += 1;
340
+ }
341
+ else if (element && element.to === undefined && (element.depth -= 1) === 0) {
342
+ element.to = tag.gt + 1;
343
+ if (tag.name === "article")
344
+ break;
345
+ }
346
+ }
347
+ for (const name of MAIN) {
348
+ const element = found.get(name);
349
+ if (element)
350
+ return html.slice(element.from, element.to ?? html.length);
351
+ }
352
+ return html;
353
+ }
354
+ /** One pass over a page's tags for its frames, in steps so a caller on the event loop can yield between them: each
355
+ * `iframe` element's, and each `frame` element's inside a `frameset` (a `<frame>` outside one is no element, as in a
356
+ * browser), `src` (decoded; null without one), in page order, with the text inside an iframe element (the fallback a
357
+ * browser does not render, where a block page keeps its message: its tags skipped, its entities decoded, read up to
358
+ * `readChars` characters); and the page's first `<base href>`. Inside an iframe element, only its end tag is a tag. */
359
+ export class FrameScan {
360
+ html;
361
+ decode;
362
+ readChars;
363
+ #at = 0;
364
+ #frameset = 0;
365
+ #base = null;
366
+ #open = null;
367
+ #frames = [];
368
+ #next;
369
+ constructor(html, decode, readChars) {
370
+ this.html = html;
371
+ this.decode = decode;
372
+ this.readChars = readChars;
373
+ this.#next = tagsOf(html);
374
+ }
375
+ get done() { return this.#at >= this.html.length; }
376
+ /** Scans the tags that start in the next `chars` characters (see PageScan.step). */
377
+ step(chars) {
378
+ const { html } = this;
379
+ const until = Math.min(html.length, this.#at + chars);
380
+ while (this.#at < until) {
381
+ const tag = this.#next(this.#at);
382
+ if (!tag) {
383
+ this.#words(html.slice(this.#at));
384
+ this.#at = html.length;
385
+ break;
386
+ }
387
+ this.#words(html.slice(this.#at, tag.lt));
388
+ if (tag.gt < 0) {
389
+ this.#at = html.length;
390
+ break;
391
+ }
392
+ this.#at = tag.gt + 1;
393
+ const { name, end, lt, gt } = tag;
394
+ if (this.#open) {
395
+ if (end && name === "iframe")
396
+ this.#close();
397
+ continue;
398
+ }
399
+ if (name === "frameset")
400
+ this.#frameset = Math.max(0, this.#frameset + (end ? -1 : 1));
401
+ else if (end)
402
+ continue;
403
+ else if (name === "base" && this.#base === null)
404
+ this.#base = attrOf(insideOf(html, name, lt, gt), "href", this.decode);
405
+ else if (name === "iframe")
406
+ this.#open = { src: attrOf(insideOf(html, name, lt, gt), "src", this.decode), text: [], read: 0 };
407
+ else if (name === "frame" && this.#frameset > 0)
408
+ this.#frames.push({ src: attrOf(insideOf(html, name, lt, gt), "src", this.decode), text: "" });
409
+ }
410
+ if (this.done && this.#open)
411
+ this.#close();
412
+ return this.done;
413
+ }
414
+ #words(raw) {
415
+ const open = this.#open;
416
+ if (!open || !raw || open.read >= this.readChars)
417
+ return;
418
+ const part = raw.slice(0, this.readChars - open.read);
419
+ open.read += part.length;
420
+ open.text.push(part.includes("&") ? this.decode(part) : part);
421
+ }
422
+ #close() {
423
+ this.#frames.push({ src: this.#open.src, text: this.#open.text.join("") });
424
+ this.#open = null;
425
+ }
426
+ /** The frames scanned, and the first base href (null without one). */
427
+ result() {
428
+ return { frames: [...this.#frames], base: this.#base };
429
+ }
430
+ }
431
+ /** One pass over a page's tags, in steps so a caller on the event loop can yield between them: its title (the first
432
+ * title element's), the http(s) URLs its `a` elements link to (each resolved against the page's first `<base href>`,
433
+ * or `url`, without its fragment, once, in page order) and, when asked for, its plain text: the text between its tags,
434
+ * entities decoded and whitespace collapsed, a block's start and end a paragraph, a row's start or a `br` a line, and a
435
+ * row's cells joined with " | ". A "<" that starts no tag is text (tagsOf); one with no ">" after it ends the page. */
436
+ export class PageScan {
437
+ html;
438
+ url;
439
+ decode;
440
+ #at = 0;
441
+ #title = null;
442
+ #base = null;
443
+ #hrefs = [];
444
+ #text;
445
+ #next;
446
+ constructor(html, url, decode, withText) {
447
+ this.html = html;
448
+ this.url = url;
449
+ this.decode = decode;
450
+ this.#text = withText ? [] : null;
451
+ this.#next = tagsOf(html);
452
+ }
453
+ /** Whether the whole page is scanned. */
454
+ get done() { return this.#at >= this.html.length; }
455
+ /** Scans the tags that start in the next `chars` characters (a tag that starts in them is read whole). */
456
+ step(chars) {
457
+ const { html } = this;
458
+ const until = Math.min(html.length, this.#at + chars);
459
+ while (this.#at < until) {
460
+ const tag = this.#next(this.#at);
461
+ if (!tag) {
462
+ this.#words(html.slice(this.#at));
463
+ this.#at = html.length;
464
+ break;
465
+ }
466
+ this.#words(html.slice(this.#at, tag.lt));
467
+ if (tag.gt < 0) {
468
+ this.#at = html.length;
469
+ break;
470
+ }
471
+ this.#at = tag.gt + 1;
472
+ if (tag.name)
473
+ this.#tag(tag.name, tag.end, tag.lt, tag.gt);
474
+ }
475
+ return this.done;
476
+ }
477
+ #tag(name, end, lt, gt) {
478
+ const text = this.#text;
479
+ if (text) {
480
+ if (BLOCKS.has(name))
481
+ text.push("\n\n");
482
+ else if (LINES.has(name))
483
+ text.push("\n");
484
+ else if (CELLS.has(name) && !end)
485
+ text.push(CELL);
486
+ }
487
+ if (end)
488
+ return;
489
+ if (name === "a" || (name === "base" && this.#base === null)) {
490
+ const href = attrOf(insideOf(this.html, name, lt, gt), "href", this.decode);
491
+ if (href === null)
492
+ return;
493
+ if (name === "a")
494
+ this.#hrefs.push(href);
495
+ else
496
+ this.#base = href;
497
+ }
498
+ else if (name === "title" && this.#title === null) {
499
+ // The first title's text is the page's title, never its text: it is read up to its end tag, and the scan goes on
500
+ // there. Only the first is looked for (once), so a page of unclosed titles is not searched to its end at each.
501
+ TITLE_END.lastIndex = gt + 1;
502
+ const close = TITLE_END.exec(this.html);
503
+ const stop = close ? close.index : this.html.indexOf("<", gt + 1);
504
+ this.#title = this.decode(this.html.slice(gt + 1, stop < 0 ? this.html.length : stop)).trim();
505
+ if (close)
506
+ this.#at = Math.max(this.#at, close.index);
507
+ }
508
+ }
509
+ #words(raw) {
510
+ if (!this.#text || !raw)
511
+ return;
512
+ // A NUL is a space here: it would read as CELL.
513
+ this.#text.push((raw.includes("&") ? this.decode(raw) : raw).replace(/[\s\u0000]+/g, " "));
514
+ }
515
+ /** The title, the links and the plain text ("" when it was not asked for) of what was scanned. */
516
+ result() {
517
+ let base = this.url;
518
+ if (this.#base !== null) {
519
+ try {
520
+ base = new URL(this.#base, this.url).href;
521
+ }
522
+ catch { /* a base that does not parse: the page's own URL stands */ }
523
+ }
524
+ const links = new Set();
525
+ for (const href of this.#hrefs) {
526
+ const link = URL.parse(href, base);
527
+ if (!link || !/^https?:$/.test(link.protocol))
528
+ continue;
529
+ link.hash = "";
530
+ links.add(link.href);
531
+ }
532
+ const text = this.#text ? this.#text.join("")
533
+ .replace(/ {2,}/g, " ")
534
+ .replace(/(^|\n) ?\u0000/g, "$1")
535
+ .replace(/ ?\u0000 ?/g, " | ")
536
+ .replace(/ {2,}/g, " ")
537
+ .replace(/ ?\n ?/g, "\n")
538
+ .replace(/\n{3,}/g, "\n\n")
539
+ .trim() : "";
540
+ return { title: this.#title ?? "", links: [...links], text };
541
+ }
542
+ }
@@ -30,10 +30,3 @@ export type PublicAnswer = {
30
30
  * with the signal's reason), and an aborted hop is never dialled.
31
31
  */
32
32
  export declare function publicFetch(url: string, options?: PublicFetchOptions): Promise<PublicAnswer>;
33
- /** An HTML page's title, its text as markdown, and the absolute http(s) URLs it links to (resolved against its
34
- * `<base href>` or `url`), each once, in page order. */
35
- export declare function readPage(html: string, url: string): Promise<{
36
- title: string;
37
- text: string;
38
- links: string[];
39
- }>;