@parcha/pi-browser 0.1.0-beta.28 → 0.1.0-beta.30

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,151 @@
1
+ // A page's HTML as markdown, in time that grows with the page's length. node-html-markdown is quadratic twice over: it
2
+ // appends to one string and reads that string's end after each append, and V8 pays for each read with a copy of the
3
+ // whole string (go.dev/dl, 3.15 MB of tables: 300 KB took 1.3 s, 600 KB 8.7 s, the whole did not end in 240 s); and
4
+ // the parser under it (node-html-parser) walks a node's siblings each time it appends one, so a table of 8,000 rows
5
+ // takes a second to parse, and four more when it is left unclosed and the parser takes it apart. So the page is never
6
+ // parsed whole: one linear scan of its tags (tagsOf), with a stack of the elements open, cuts it into pieces of about
7
+ // PIECE_CHARS, each converted on its own. A piece is cut before a block element's start tag where every element open
8
+ // adds nothing to the markdown (a div, a section); past WIDE_CHARS, also between a list's items or a table's rows, or
9
+ // before a <br> outside a table's rows, so a table or a list smaller than a piece is never cut; past FORCE_CHARS, before
10
+ // any tag. Each piece starts with the start tags of the elements open around it that put something into the markdown
11
+ // (a list, a table, a quote; a table's led by its first row, which a markdown table takes for its header) and ends with
12
+ // their end tags, so no piece is ever unclosed. The markdown of a well-formed page in pieces is the one pass's, but for
13
+ // the spaces that end a line at a piece's edges; an ordered list in pieces is counted from 1 in each.
14
+ import { NodeHtmlMarkdown } from "node-html-markdown";
15
+ import { tagsOf } from "./page-scan.js";
16
+ export const PIECE_CHARS = 16_384;
17
+ /** Past this, a piece is cut between a list's items or a table's rows, or before a `<br>`. */
18
+ const WIDE_CHARS = 2 * PIECE_CHARS;
19
+ /** Past this, a piece is cut before any tag: what is open is closed around it and reopened after. */
20
+ const FORCE_CHARS = 3 * PIECE_CHARS;
21
+ const CELLS = new Set(["tr", "td", "th"]);
22
+ /** Containers that add nothing of their own to the markdown: never reopened around a piece. */
23
+ const NEUTRAL = new Set(["html", "body", "main", "article", "section", "div", "header", "footer", "nav", "aside", "form", "center", "fieldset", "figure", "hgroup", "span", "font", "address", "details"]);
24
+ /** Containers a piece may be cut inside, between their children (each reopened around the piece). */
25
+ const BETWEEN = new Set(["ul", "ol", "menu", "dl", "blockquote", "table", "thead", "tbody", "tfoot"]);
26
+ /** node-html-markdown's block elements: a piece is cut only before one's start tag. */
27
+ const BLOCK = new Set(["address", "article", "aside", "audio", "blockquote", "body", "canvas", "center", "dd", "dir", "div", "dl", "dt", "fieldset", "figcaption", "figure", "footer", "form", "frameset", "h1", "h2", "h3", "h4", "h5", "h6", "header", "hgroup", "hr", "html", "isindex", "li", "main", "menu", "nav", "noframes", "noscript", "ol", "output", "p", "pre", "section", "table", "tbody", "td", "tfoot", "th", "thead", "tr", "ul"]);
28
+ const VOID = new Set(["area", "base", "br", "col", "embed", "frame", "hr", "img", "input", "keygen", "link", "meta", "param", "source", "track", "wbr"]);
29
+ /** For a start tag: the open element it ends, if one is open below the nearest of the elements that bound it. */
30
+ const ENDS = {
31
+ li: { ends: new Set(["li"]), within: new Set(["ul", "ol", "menu"]) },
32
+ dt: { ends: new Set(["dt", "dd"]), within: new Set(["dl"]) },
33
+ dd: { ends: new Set(["dt", "dd"]), within: new Set(["dl"]) },
34
+ tr: { ends: new Set(["tr"]), within: new Set(["table", "thead", "tbody", "tfoot"]) },
35
+ td: { ends: new Set(["td", "th"]), within: new Set(["tr", "table"]) },
36
+ th: { ends: new Set(["td", "th"]), within: new Set(["tr", "table"]) },
37
+ thead: { ends: new Set(["thead", "tbody", "tfoot"]), within: new Set(["table"]) },
38
+ tbody: { ends: new Set(["thead", "tbody", "tfoot"]), within: new Set(["table"]) },
39
+ tfoot: { ends: new Set(["thead", "tbody", "tfoot"]), within: new Set(["table"]) },
40
+ option: { ends: new Set(["option"]), within: new Set(["select", "datalist", "optgroup"]) },
41
+ body: { ends: new Set(["head"]), within: new Set(["html"]) },
42
+ };
43
+ /** How far down the stack an end tag, or a start tag that ends an element, looks for its element. */
44
+ const REACH = 64;
45
+ /** The most elements reopened around a piece. A start tag longer than MAX_TAG is reopened bare (`<ul>`), and a
46
+ * table's first row longer than a quarter piece is not repeated. */
47
+ const MAX_REOPENED = 32;
48
+ const MAX_TAG = 256;
49
+ /** `html` in pieces, each well-formed on its own (see the file's head). */
50
+ export function piecesOf(html) {
51
+ const pieces = [];
52
+ const stack = [];
53
+ /** How many open elements a cut would split (neither NEUTRAL nor BETWEEN); how many are BETWEEN; how many a row or
54
+ * its cells. */
55
+ let inside = 0;
56
+ let between = 0;
57
+ let cells = 0;
58
+ const count = (name, by) => {
59
+ if (BETWEEN.has(name))
60
+ between += by;
61
+ else if (!NEUTRAL.has(name))
62
+ inside += by;
63
+ if (CELLS.has(name))
64
+ cells += by;
65
+ };
66
+ let start = 0;
67
+ let opening = "";
68
+ const reopened = () => stack.filter((open) => !NEUTRAL.has(open.name)).slice(-MAX_REOPENED);
69
+ const cut = (at) => {
70
+ if (at <= start)
71
+ return;
72
+ pieces.push(opening + html.slice(start, at) + reopened().reverse().map((open) => `</${open.name}>`).join(""));
73
+ start = at;
74
+ opening = reopened().map((open) => open.tag + (open.header && open.header.length <= PIECE_CHARS / 4 ? open.header : "")).join("");
75
+ };
76
+ /** Ends the elements from `index` up, those above it at `before`, the one at `index` at `after` (past its own end
77
+ * tag, when that is what ends it). */
78
+ const pop = (index, before, after) => {
79
+ while (stack.length > index) {
80
+ const open = stack.pop();
81
+ count(open.name, -1);
82
+ const at = stack.length === index ? after : before;
83
+ // A row its own end tag ended holds that tag; one another tag ended is closed here.
84
+ if (open.rowOf && open.rowOf.header === undefined)
85
+ open.rowOf.header = html.slice(open.at, at) + (at === after && after !== before ? "" : "</tr>");
86
+ }
87
+ };
88
+ /** The index of the nearest open element `name` (or one in `names`), looking no further down than REACH or an
89
+ * element in `within`; -1 when there is none. */
90
+ const find = (names, within) => {
91
+ for (let i = stack.length - 1; i >= Math.max(0, stack.length - REACH); i -= 1) {
92
+ const open = stack[i].name;
93
+ if (typeof names === "string" ? open === names : names.has(open))
94
+ return i;
95
+ if (within?.has(open))
96
+ return -1;
97
+ }
98
+ return -1;
99
+ };
100
+ const next = tagsOf(html);
101
+ for (let tag = next(0); tag && tag.gt >= 0; tag = next(tag.gt + 1)) {
102
+ const { lt, gt, name, end } = tag;
103
+ if (!name)
104
+ continue;
105
+ if (end) {
106
+ if (lt - start >= FORCE_CHARS)
107
+ cut(lt);
108
+ const at = find(name);
109
+ if (at >= 0)
110
+ pop(at, lt, gt + 1);
111
+ continue;
112
+ }
113
+ const rule = ENDS[name];
114
+ if (rule) {
115
+ const at = find(rule.ends, rule.within);
116
+ if (at >= 0)
117
+ pop(at, lt, lt);
118
+ }
119
+ if (BLOCK.has(name) && stack.at(-1)?.name === "p")
120
+ pop(stack.length - 1, lt, lt);
121
+ const size = lt - start;
122
+ if (size >= FORCE_CHARS || (BLOCK.has(name) && inside === 0 && size >= (between ? WIDE_CHARS : PIECE_CHARS)) || (name === "br" && cells === 0 && size >= WIDE_CHARS))
123
+ cut(lt);
124
+ if (VOID.has(name) || html.charCodeAt(gt - 1) === 47)
125
+ continue;
126
+ const open = { name, tag: gt - lt <= MAX_TAG ? html.slice(lt, gt + 1) : `<${name}>` };
127
+ if (name === "tr") {
128
+ const table = stack[find("table")];
129
+ if (table && !table.row) {
130
+ table.row = true;
131
+ open.rowOf = table;
132
+ open.at = lt;
133
+ }
134
+ }
135
+ stack.push(open);
136
+ count(name, 1);
137
+ }
138
+ cut(html.length);
139
+ return pieces;
140
+ }
141
+ /** `html` as markdown, a piece at a time (see the file's head). */
142
+ export function markdownOf(html) {
143
+ const converter = new NodeHtmlMarkdown({ keepDataImages: false });
144
+ const parts = [];
145
+ for (const piece of piecesOf(html)) {
146
+ const markdown = converter.translate(piece).replace(/^\n+|\n+$/g, "");
147
+ if (markdown.trim())
148
+ parts.push(markdown);
149
+ }
150
+ return parts.join("\n\n").trim();
151
+ }
@@ -0,0 +1,95 @@
1
+ /** An entity decoder for a run of text or an attribute value: node-html-parser's in the worker, entities.ts's on the
2
+ * event loop. */
3
+ export type Decode = (text: string) => string;
4
+ /** A page without its script, style, svg, noscript and template elements and its comments, in steps so a caller on the
5
+ * event loop can yield between them (dropInert, all at once). One whose end is missing runs to the end of the page, as
6
+ * a browser reads it; but an svg's start tag goes alone, since a browser lets the HTML that follows out of an unclosed
7
+ * svg. A template ends at the end tag that closes it, as in a browser: a template inside it is counted, so its end tag
8
+ * ends only the inner one, and the text of an element that holds text, not markup, is never read as tags in it
9
+ * (TEXT_ONLY: a script, a textarea, a title). Out of a template, a textarea's or a title's text is not read as tags
10
+ * either: a "<template>" or a "<script>" in it is text, kept with each "<" written "&lt;" (KEPT_TEXT). A nested svg ends at the first end tag of its name. A search for an end tag that finds none is not made again from
11
+ * a later start of that name (none follows it either), so a page of unclosed svgs is read once, not once for each; and
12
+ * the templates' ends are found in one forward pass however many there are (#templateEnd). */
13
+ export declare class InertDrop {
14
+ #private;
15
+ readonly html: string;
16
+ constructor(html: string);
17
+ /** Drops the blocks found in the next `chars` characters or so; true once the whole page is read. A template's end
18
+ * is looked for `chars` characters at a time too, so no step reads much more than that. */
19
+ step(chars: number): boolean;
20
+ /** The page as dropped so far (the whole of it once step has answered true). */
21
+ result(): string;
22
+ }
23
+ /** `html` without its inert blocks, all at once (see InertDrop). */
24
+ export declare function dropInert(html: string): string;
25
+ /** `html` cut to at most `max` characters, after the last tag that ends before the cut, so no tag is left half; at
26
+ * `max` itself when that tag ends more than 64 KiB before it (a page of one long text), or one before it when `max`
27
+ * falls between the two halves of a character written as a surrogate pair, so no half of one is left. */
28
+ export declare function cutHtml(html: string, max: number): string;
29
+ /** A tag in a page: `lt` and `gt` the offsets of its "<" and ">", `name` lowercase ("" for a declaration, a comment
30
+ * or an instruction), `end` for an end tag; `gt` is -1 for a "<" that starts a tag with no ">" after it. */
31
+ export type Tag = {
32
+ lt: number;
33
+ gt: number;
34
+ name: string;
35
+ end: boolean;
36
+ };
37
+ /**
38
+ * A reader of `html`'s tags, each call the next tag from `at`: "<" then a letter starts one, "</" an end tag, "<!" a
39
+ * declaration or a comment (which ends at its "-->") and "<?" an instruction; any other "<" is text, and passed. Null
40
+ * when none is left. A ">" inside a quoted attribute value (`title="x > y"`) does not end the tag. A quote that no quote
41
+ * closes is no quote: the tag ends at its first ">", so an open quote never takes the rest of the page. Each search
42
+ * remembers what it found, so a scan whose `at` moves forward reads the page once whatever its markup (no "=" for
43
+ * pages, or a thousand open quotes).
44
+ */
45
+ export declare function tagsOf(html: string): (at: number) => Tag | null;
46
+ /** The first article element of `html`, else its first main, else its body, as HTML from its start tag to its end tag
47
+ * (or the page's end); the whole of `html` when it has none of them. One pass: an element of the same name inside one
48
+ * is counted, so its end tag does not end the one around it. Run on a page its inert blocks are dropped from, so
49
+ * none is picked from inside a script or a comment. */
50
+ export declare function mainOf(html: string): string;
51
+ /** One pass over a page's tags for its frames, in steps so a caller on the event loop can yield between them: each
52
+ * `iframe` element's, and each `frame` element's inside a `frameset` (a `<frame>` outside one is no element, as in a
53
+ * browser), `src` (decoded; null without one), in page order, with the text inside an iframe element (the fallback a
54
+ * browser does not render, where a block page keeps its message: its tags skipped, its entities decoded, read up to
55
+ * `readChars` characters); and the page's first `<base href>`. Inside an iframe element, only its end tag is a tag. */
56
+ export declare class FrameScan {
57
+ #private;
58
+ readonly html: string;
59
+ readonly decode: Decode;
60
+ readonly readChars: number;
61
+ constructor(html: string, decode: Decode, readChars: number);
62
+ get done(): boolean;
63
+ /** Scans the tags that start in the next `chars` characters (see PageScan.step). */
64
+ step(chars: number): boolean;
65
+ /** The frames scanned, and the first base href (null without one). */
66
+ result(): {
67
+ frames: Array<{
68
+ src: string | null;
69
+ text: string;
70
+ }>;
71
+ base: string | null;
72
+ };
73
+ }
74
+ /** One pass over a page's tags, in steps so a caller on the event loop can yield between them: its title (the first
75
+ * title element's), the http(s) URLs its `a` elements link to (each resolved against the page's first `<base href>`,
76
+ * or `url`, without its fragment, once, in page order) and, when asked for, its plain text: the text between its tags,
77
+ * entities decoded and whitespace collapsed, a block's start and end a paragraph, a row's start or a `br` a line, and a
78
+ * row's cells joined with " | ". A "<" that starts no tag is text (tagsOf); one with no ">" after it ends the page. */
79
+ export declare class PageScan {
80
+ #private;
81
+ readonly html: string;
82
+ readonly url: string;
83
+ readonly decode: Decode;
84
+ constructor(html: string, url: string, decode: Decode, withText: boolean);
85
+ /** Whether the whole page is scanned. */
86
+ get done(): boolean;
87
+ /** Scans the tags that start in the next `chars` characters (a tag that starts in them is read whole). */
88
+ step(chars: number): boolean;
89
+ /** The title, the links and the plain text ("" when it was not asked for) of what was scanned. */
90
+ result(): {
91
+ title: string;
92
+ links: string[];
93
+ text: string;
94
+ };
95
+ }