@linxiraos/pi-utils 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +449 -0
- package/README.md +36 -0
- package/dist/types/abortable.d.ts +32 -0
- package/dist/types/acp/connection.d.ts +118 -0
- package/dist/types/acp/protocol.d.ts +526 -0
- package/dist/types/acp/schema.d.ts +41 -0
- package/dist/types/acp/stream.d.ts +8 -0
- package/dist/types/acp/transport.d.ts +81 -0
- package/dist/types/acp.d.ts +6 -0
- package/dist/types/async.d.ts +20 -0
- package/dist/types/binary.d.ts +17 -0
- package/dist/types/browsers.d.ts +68 -0
- package/dist/types/chalk.d.ts +125 -0
- package/dist/types/cli.d.ts +134 -0
- package/dist/types/color.d.ts +102 -0
- package/dist/types/dates.d.ts +7 -0
- package/dist/types/dirs.d.ts +282 -0
- package/dist/types/docx/converter.d.ts +46 -0
- package/dist/types/docx/xml.d.ts +26 -0
- package/dist/types/docx/zip.d.ts +6 -0
- package/dist/types/docx.d.ts +11 -0
- package/dist/types/dom/core.d.ts +431 -0
- package/dist/types/dom/parser.d.ts +7 -0
- package/dist/types/dom/selector.d.ts +5 -0
- package/dist/types/dom.d.ts +5 -0
- package/dist/types/env.d.ts +124 -0
- package/dist/types/fetch-retry.d.ts +96 -0
- package/dist/types/file-lock.d.ts +23 -0
- package/dist/types/format.d.ts +37 -0
- package/dist/types/frontmatter.d.ts +46 -0
- package/dist/types/fs-error.d.ts +31 -0
- package/dist/types/glob.d.ts +28 -0
- package/dist/types/headers.d.ts +34 -0
- package/dist/types/index.d.ts +39 -0
- package/dist/types/json-parse.d.ts +74 -0
- package/dist/types/json.d.ts +14 -0
- package/dist/types/logger/rotating-file.d.ts +18 -0
- package/dist/types/logger.d.ts +96 -0
- package/dist/types/loop-phase.d.ts +10 -0
- package/dist/types/lru.d.ts +46 -0
- package/dist/types/marked/core.d.ts +445 -0
- package/dist/types/marked.d.ts +2 -0
- package/dist/types/mermaid-ascii.d.ts +11 -0
- package/dist/types/mime.d.ts +29 -0
- package/dist/types/module-timer.d.ts +1 -0
- package/dist/types/path-tree.d.ts +76 -0
- package/dist/types/path.d.ts +2 -0
- package/dist/types/peek-file.d.ts +29 -0
- package/dist/types/postmortem.d.ts +104 -0
- package/dist/types/process-name.d.ts +7 -0
- package/dist/types/procmgr.d.ts +74 -0
- package/dist/types/prompt.d.ts +18 -0
- package/dist/types/ptree.d.ts +110 -0
- package/dist/types/readability/readability.d.ts +9 -0
- package/dist/types/readability/readerable.d.ts +10 -0
- package/dist/types/readability/types.d.ts +70 -0
- package/dist/types/readability.d.ts +4 -0
- package/dist/types/ring.d.ts +93 -0
- package/dist/types/runtime-install.d.ts +78 -0
- package/dist/types/sanitize-text.d.ts +29 -0
- package/dist/types/snowflake.d.ts +25 -0
- package/dist/types/stderr-guard.d.ts +22 -0
- package/dist/types/stream.d.ts +67 -0
- package/dist/types/tab-spacing.d.ts +24 -0
- package/dist/types/temp.d.ts +17 -0
- package/dist/types/template.d.ts +62 -0
- package/dist/types/timing-buffer.d.ts +22 -0
- package/dist/types/tls-fetch.d.ts +37 -0
- package/dist/types/turndown/gfm.d.ts +11 -0
- package/dist/types/turndown/html.d.ts +5 -0
- package/dist/types/turndown/service.d.ts +21 -0
- package/dist/types/turndown/types.d.ts +70 -0
- package/dist/types/turndown.d.ts +4 -0
- package/dist/types/type-guards.d.ts +5 -0
- package/dist/types/vendor/mermaid-ascii/ascii/ansi.d.ts +41 -0
- package/dist/types/vendor/mermaid-ascii/ascii/canvas.d.ts +89 -0
- package/dist/types/vendor/mermaid-ascii/ascii/class-diagram.d.ts +7 -0
- package/dist/types/vendor/mermaid-ascii/ascii/converter.d.ts +12 -0
- package/dist/types/vendor/mermaid-ascii/ascii/draw.d.ts +66 -0
- package/dist/types/vendor/mermaid-ascii/ascii/edge-bundling.d.ts +48 -0
- package/dist/types/vendor/mermaid-ascii/ascii/edge-routing.d.ts +43 -0
- package/dist/types/vendor/mermaid-ascii/ascii/er-diagram.d.ts +7 -0
- package/dist/types/vendor/mermaid-ascii/ascii/grid.d.ts +56 -0
- package/dist/types/vendor/mermaid-ascii/ascii/index.d.ts +65 -0
- package/dist/types/vendor/mermaid-ascii/ascii/multiline-utils.d.ts +27 -0
- package/dist/types/vendor/mermaid-ascii/ascii/pathfinder.d.ts +17 -0
- package/dist/types/vendor/mermaid-ascii/ascii/sequence.d.ts +7 -0
- package/dist/types/vendor/mermaid-ascii/ascii/shapes/circle.d.ts +11 -0
- package/dist/types/vendor/mermaid-ascii/ascii/shapes/corners.d.ts +34 -0
- package/dist/types/vendor/mermaid-ascii/ascii/shapes/diamond.d.ts +11 -0
- package/dist/types/vendor/mermaid-ascii/ascii/shapes/hexagon.d.ts +11 -0
- package/dist/types/vendor/mermaid-ascii/ascii/shapes/index.d.ts +26 -0
- package/dist/types/vendor/mermaid-ascii/ascii/shapes/rectangle.d.ts +31 -0
- package/dist/types/vendor/mermaid-ascii/ascii/shapes/rounded.d.ts +11 -0
- package/dist/types/vendor/mermaid-ascii/ascii/shapes/special.d.ts +59 -0
- package/dist/types/vendor/mermaid-ascii/ascii/shapes/stadium.d.ts +17 -0
- package/dist/types/vendor/mermaid-ascii/ascii/shapes/state.d.ts +30 -0
- package/dist/types/vendor/mermaid-ascii/ascii/shapes/types.d.ts +55 -0
- package/dist/types/vendor/mermaid-ascii/ascii/types.d.ts +206 -0
- package/dist/types/vendor/mermaid-ascii/ascii/validate.d.ts +51 -0
- package/dist/types/vendor/mermaid-ascii/ascii/xychart.d.ts +2 -0
- package/dist/types/vendor/mermaid-ascii/class/parser.d.ts +6 -0
- package/dist/types/vendor/mermaid-ascii/class/types.d.ts +102 -0
- package/dist/types/vendor/mermaid-ascii/er/parser.d.ts +6 -0
- package/dist/types/vendor/mermaid-ascii/er/types.d.ts +76 -0
- package/dist/types/vendor/mermaid-ascii/index.d.ts +1 -0
- package/dist/types/vendor/mermaid-ascii/multiline-utils.d.ts +9 -0
- package/dist/types/vendor/mermaid-ascii/parser.d.ts +7 -0
- package/dist/types/vendor/mermaid-ascii/sequence/parser.d.ts +6 -0
- package/dist/types/vendor/mermaid-ascii/sequence/types.d.ts +130 -0
- package/dist/types/vendor/mermaid-ascii/text-metrics.d.ts +21 -0
- package/dist/types/vendor/mermaid-ascii/types.d.ts +114 -0
- package/dist/types/vendor/mermaid-ascii/xychart/colors.d.ts +25 -0
- package/dist/types/vendor/mermaid-ascii/xychart/parser.d.ts +6 -0
- package/dist/types/vendor/mermaid-ascii/xychart/types.d.ts +145 -0
- package/dist/types/version.d.ts +18 -0
- package/dist/types/vterm/buffer.d.ts +99 -0
- package/dist/types/vterm/terminal.d.ts +44 -0
- package/dist/types/vterm.d.ts +8 -0
- package/dist/types/which.d.ts +37 -0
- package/dist/types/worker-host.d.ts +51 -0
- package/dist/types/xml.d.ts +31 -0
- package/package.json +59 -0
- package/src/abortable.ts +109 -0
- package/src/acp/connection.ts +344 -0
- package/src/acp/protocol.ts +466 -0
- package/src/acp/schema.ts +160 -0
- package/src/acp/stream.ts +82 -0
- package/src/acp/transport.ts +213 -0
- package/src/acp.ts +6 -0
- package/src/async.ts +91 -0
- package/src/binary.ts +50 -0
- package/src/browsers.ts +501 -0
- package/src/chalk.ts +312 -0
- package/src/cli.ts +529 -0
- package/src/color.ts +302 -0
- package/src/dates.ts +194 -0
- package/src/dirs.ts +1026 -0
- package/src/docx/converter.ts +681 -0
- package/src/docx/xml.ts +166 -0
- package/src/docx/zip.ts +87 -0
- package/src/docx.ts +20 -0
- package/src/dom/core.ts +1254 -0
- package/src/dom/parser.ts +370 -0
- package/src/dom/selector.ts +290 -0
- package/src/dom.ts +33 -0
- package/src/env.ts +400 -0
- package/src/fetch-retry.ts +379 -0
- package/src/file-lock.ts +69 -0
- package/src/format.ts +113 -0
- package/src/frontmatter.ts +205 -0
- package/src/fs-error.ts +56 -0
- package/src/glob.ts +189 -0
- package/src/headers.ts +167 -0
- package/src/index.ts +59 -0
- package/src/json-parse.ts +821 -0
- package/src/json.ts +23 -0
- package/src/logger/rotating-file.ts +149 -0
- package/src/logger.ts +798 -0
- package/src/loop-phase.ts +49 -0
- package/src/lru.ts +185 -0
- package/src/marked/core.ts +1576 -0
- package/src/marked.ts +2 -0
- package/src/mermaid-ascii.ts +31 -0
- package/src/mime.ts +159 -0
- package/src/module-timer.ts +148 -0
- package/src/path-tree.ts +147 -0
- package/src/path.ts +28 -0
- package/src/peek-file.ts +188 -0
- package/src/postmortem.ts +433 -0
- package/src/process-name.ts +57 -0
- package/src/procmgr.ts +247 -0
- package/src/prompt.ts +545 -0
- package/src/ptree.ts +412 -0
- package/src/readability/readability.ts +533 -0
- package/src/readability/readerable.ts +51 -0
- package/src/readability/types.ts +72 -0
- package/src/readability.ts +11 -0
- package/src/ring.ts +169 -0
- package/src/runtime-install.ts +393 -0
- package/src/sanitize-text.ts +96 -0
- package/src/snowflake.ts +121 -0
- package/src/stderr-guard.ts +155 -0
- package/src/stream.ts +454 -0
- package/src/tab-spacing.ts +336 -0
- package/src/temp.ts +134 -0
- package/src/template.ts +586 -0
- package/src/timing-buffer.ts +47 -0
- package/src/tls-fetch.ts +178 -0
- package/src/turndown/gfm.ts +106 -0
- package/src/turndown/html.ts +257 -0
- package/src/turndown/service.ts +334 -0
- package/src/turndown/types.ts +81 -0
- package/src/turndown.ts +5 -0
- package/src/type-guards.ts +17 -0
- package/src/vendor/mermaid-ascii/NOTICE +33 -0
- package/src/vendor/mermaid-ascii/ascii/ansi.ts +409 -0
- package/src/vendor/mermaid-ascii/ascii/canvas.ts +476 -0
- package/src/vendor/mermaid-ascii/ascii/class-diagram.ts +699 -0
- package/src/vendor/mermaid-ascii/ascii/converter.ts +271 -0
- package/src/vendor/mermaid-ascii/ascii/draw.ts +1382 -0
- package/src/vendor/mermaid-ascii/ascii/edge-bundling.ts +328 -0
- package/src/vendor/mermaid-ascii/ascii/edge-routing.ts +297 -0
- package/src/vendor/mermaid-ascii/ascii/er-diagram.ts +441 -0
- package/src/vendor/mermaid-ascii/ascii/grid.ts +578 -0
- package/src/vendor/mermaid-ascii/ascii/index.ts +187 -0
- package/src/vendor/mermaid-ascii/ascii/multiline-utils.ts +78 -0
- package/src/vendor/mermaid-ascii/ascii/pathfinder.ts +277 -0
- package/src/vendor/mermaid-ascii/ascii/sequence.ts +460 -0
- package/src/vendor/mermaid-ascii/ascii/shapes/circle.ts +27 -0
- package/src/vendor/mermaid-ascii/ascii/shapes/corners.ts +127 -0
- package/src/vendor/mermaid-ascii/ascii/shapes/diamond.ts +27 -0
- package/src/vendor/mermaid-ascii/ascii/shapes/hexagon.ts +27 -0
- package/src/vendor/mermaid-ascii/ascii/shapes/index.ts +101 -0
- package/src/vendor/mermaid-ascii/ascii/shapes/rectangle.ts +175 -0
- package/src/vendor/mermaid-ascii/ascii/shapes/rounded.ts +27 -0
- package/src/vendor/mermaid-ascii/ascii/shapes/special.ts +296 -0
- package/src/vendor/mermaid-ascii/ascii/shapes/stadium.ts +114 -0
- package/src/vendor/mermaid-ascii/ascii/shapes/state.ts +192 -0
- package/src/vendor/mermaid-ascii/ascii/shapes/types.ts +73 -0
- package/src/vendor/mermaid-ascii/ascii/types.ts +273 -0
- package/src/vendor/mermaid-ascii/ascii/validate.ts +120 -0
- package/src/vendor/mermaid-ascii/ascii/xychart.ts +875 -0
- package/src/vendor/mermaid-ascii/class/parser.ts +290 -0
- package/src/vendor/mermaid-ascii/class/types.ts +121 -0
- package/src/vendor/mermaid-ascii/er/parser.ts +181 -0
- package/src/vendor/mermaid-ascii/er/types.ts +91 -0
- package/src/vendor/mermaid-ascii/index.ts +14 -0
- package/src/vendor/mermaid-ascii/multiline-utils.ts +30 -0
- package/src/vendor/mermaid-ascii/parser.ts +645 -0
- package/src/vendor/mermaid-ascii/sequence/parser.ts +207 -0
- package/src/vendor/mermaid-ascii/sequence/types.ts +146 -0
- package/src/vendor/mermaid-ascii/text-metrics.ts +71 -0
- package/src/vendor/mermaid-ascii/types.ts +164 -0
- package/src/vendor/mermaid-ascii/xychart/colors.ts +140 -0
- package/src/vendor/mermaid-ascii/xychart/parser.ts +115 -0
- package/src/vendor/mermaid-ascii/xychart/types.ts +150 -0
- package/src/version.ts +99 -0
- package/src/vterm/buffer.ts +218 -0
- package/src/vterm/terminal.ts +773 -0
- package/src/vterm.ts +8 -0
- package/src/which.ts +234 -0
- package/src/worker-host.ts +100 -0
- package/src/xml.ts +313 -0
|
@@ -0,0 +1,533 @@
|
|
|
1
|
+
/** Behavior-compatible reimplementation of @mozilla/readability's used surface. */
|
|
2
|
+
|
|
3
|
+
import type {
|
|
4
|
+
ReadabilityArticle,
|
|
5
|
+
ReadabilityDocument,
|
|
6
|
+
ReadabilityElement,
|
|
7
|
+
ReadabilityNode,
|
|
8
|
+
ReadabilityOptions,
|
|
9
|
+
} from "./types";
|
|
10
|
+
|
|
11
|
+
const UNLIKELY =
|
|
12
|
+
/-ad-|ai2html|banner|breadcrumbs|comment|community|combx|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup/i;
|
|
13
|
+
const POSSIBLE = /and|article|body|column|content|main|shadow/i;
|
|
14
|
+
const POSITIVE = /article|body|content|entry|hentry|h-entry|main|page|pagination|post|text|blog|story/i;
|
|
15
|
+
const NEGATIVE =
|
|
16
|
+
/-ad-|hidden|^hid$| hid$| hid |^hid |banner|comment|com-|contact|footer|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|widget/i;
|
|
17
|
+
const BYLINE = /byline|author|dateline|writtenby|p-author/i;
|
|
18
|
+
const SCORE_TAGS = new Set(["SECTION", "H2", "H3", "H4", "H5", "H6", "P", "TD", "PRE"]);
|
|
19
|
+
const DROP_TAGS = [
|
|
20
|
+
"form",
|
|
21
|
+
"fieldset",
|
|
22
|
+
"object",
|
|
23
|
+
"embed",
|
|
24
|
+
"footer",
|
|
25
|
+
"link",
|
|
26
|
+
"aside",
|
|
27
|
+
"iframe",
|
|
28
|
+
"input",
|
|
29
|
+
"textarea",
|
|
30
|
+
"select",
|
|
31
|
+
"button",
|
|
32
|
+
];
|
|
33
|
+
const UNLIKELY_ROLES = new Set(["menu", "menubar", "complementary", "navigation", "alert", "alertdialog", "dialog"]);
|
|
34
|
+
const ARTICLE_TYPES =
|
|
35
|
+
/^(?:Article|AdvertiserContentArticle|NewsArticle|AnalysisNewsArticle|OpinionNewsArticle|ReportageNewsArticle|ReviewNewsArticle|Report|ScholarlyArticle|MedicalScholarlyArticle|SocialMediaPosting|BlogPosting|LiveBlogPosting|DiscussionForumPosting|TechArticle|APIReference)$/;
|
|
36
|
+
const NORMALIZE = /\s{2,}/g;
|
|
37
|
+
|
|
38
|
+
type Metadata = {
|
|
39
|
+
title?: string;
|
|
40
|
+
byline?: string;
|
|
41
|
+
excerpt?: string;
|
|
42
|
+
siteName?: string;
|
|
43
|
+
publishedTime?: string | null;
|
|
44
|
+
};
|
|
45
|
+
|
|
46
|
+
type Attempt = { element: ReadabilityElement; length: number; dir?: string | null };
|
|
47
|
+
|
|
48
|
+
function elements(collection: ArrayLike<ReadabilityElement>): ReadabilityElement[] {
|
|
49
|
+
return Array.from(collection);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function descendants(root: ReadabilityElement): ReadabilityElement[] {
|
|
53
|
+
const result: ReadabilityElement[] = [];
|
|
54
|
+
const pending = elements(root.children).reverse();
|
|
55
|
+
while (pending.length) {
|
|
56
|
+
const node = pending.pop();
|
|
57
|
+
if (!node) continue;
|
|
58
|
+
result.push(node);
|
|
59
|
+
const children = elements(node.children);
|
|
60
|
+
for (let index = children.length - 1; index >= 0; index--) pending.push(children[index]);
|
|
61
|
+
}
|
|
62
|
+
return result;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function text(node: ReadabilityNode): string {
|
|
66
|
+
return (node.textContent ?? "").trim().replace(NORMALIZE, " ");
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function matchLabel(node: ReadabilityElement): string {
|
|
70
|
+
return `${typeof node.className === "string" ? node.className : ""} ${node.id ?? ""}`;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function classWeight(node: ReadabilityElement): number {
|
|
74
|
+
const label = matchLabel(node);
|
|
75
|
+
return (POSITIVE.test(label) ? 25 : 0) - (NEGATIVE.test(label) ? 25 : 0);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function initialScore(node: ReadabilityElement): number {
|
|
79
|
+
let score = classWeight(node);
|
|
80
|
+
switch (node.tagName) {
|
|
81
|
+
case "DIV":
|
|
82
|
+
score += 5;
|
|
83
|
+
break;
|
|
84
|
+
case "PRE":
|
|
85
|
+
case "TD":
|
|
86
|
+
case "BLOCKQUOTE":
|
|
87
|
+
score += 3;
|
|
88
|
+
break;
|
|
89
|
+
case "ADDRESS":
|
|
90
|
+
case "OL":
|
|
91
|
+
case "UL":
|
|
92
|
+
case "DL":
|
|
93
|
+
case "DD":
|
|
94
|
+
case "DT":
|
|
95
|
+
case "LI":
|
|
96
|
+
case "FORM":
|
|
97
|
+
score -= 3;
|
|
98
|
+
break;
|
|
99
|
+
case "H1":
|
|
100
|
+
case "H2":
|
|
101
|
+
case "H3":
|
|
102
|
+
case "H4":
|
|
103
|
+
case "H5":
|
|
104
|
+
case "H6":
|
|
105
|
+
case "TH":
|
|
106
|
+
score -= 5;
|
|
107
|
+
break;
|
|
108
|
+
}
|
|
109
|
+
return score;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
function linkDensity(node: ReadabilityElement): number {
|
|
113
|
+
const total = text(node).length;
|
|
114
|
+
if (!total) return 0;
|
|
115
|
+
let linked = 0;
|
|
116
|
+
for (const link of elements(node.getElementsByTagName("a"))) {
|
|
117
|
+
linked += text(link).length * ((link.getAttribute("href") ?? "").startsWith("#") ? 0.3 : 1);
|
|
118
|
+
}
|
|
119
|
+
return linked / total;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
function visible(node: ReadabilityElement): boolean {
|
|
123
|
+
const style = node.getAttribute("style")?.toLowerCase() ?? "";
|
|
124
|
+
return (
|
|
125
|
+
!node.hasAttribute("hidden") &&
|
|
126
|
+
node.getAttribute("aria-hidden") !== "true" &&
|
|
127
|
+
!/display\s*:\s*none|visibility\s*:\s*hidden/.test(style)
|
|
128
|
+
);
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
function removeAll(root: ReadabilityNode, tags: readonly string[]): void {
|
|
132
|
+
const container = root as ReadabilityElement;
|
|
133
|
+
for (const tag of tags) {
|
|
134
|
+
for (const node of elements(container.getElementsByTagName(tag))) node.remove();
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
function entityDecode(value: string | undefined | null): string | undefined | null {
|
|
139
|
+
if (!value) return value;
|
|
140
|
+
const named: Record<string, string> = { quot: '"', amp: "&", apos: "'", lt: "<", gt: ">" };
|
|
141
|
+
return value
|
|
142
|
+
.replace(/&(quot|amp|apos|lt|gt);/g, (_, name: string) => named[name] ?? "")
|
|
143
|
+
.replace(/&#(?:x([0-9a-f]+)|(\d+));/gi, (_, hex: string | undefined, decimal: string | undefined) => {
|
|
144
|
+
const value = Number.parseInt(hex ?? decimal ?? "0", hex ? 16 : 10);
|
|
145
|
+
return String.fromCodePoint(
|
|
146
|
+
value === 0 || value > 0x10ffff || (value >= 0xd800 && value <= 0xdfff) ? 0xfffd : value,
|
|
147
|
+
);
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
function titleFromDocument(document: ReadabilityDocument): string {
|
|
152
|
+
const titleElement = elements(document.getElementsByTagName("title"))[0];
|
|
153
|
+
const original = typeof document.title === "string" ? document.title.trim() : titleElement ? text(titleElement) : "";
|
|
154
|
+
let title = original;
|
|
155
|
+
const separators = [...original.matchAll(/ [|\\/>»-] /g)];
|
|
156
|
+
if (separators.length) {
|
|
157
|
+
title = original.slice(0, separators.at(-1)?.index);
|
|
158
|
+
if (title.trim().split(/\s+/).length < 3) title = original.replace(/^[^|\\/>»-]*[|\\/>»-]/, "");
|
|
159
|
+
} else if (title.includes(": ")) {
|
|
160
|
+
const matchingHeading = elements(document.querySelectorAll("h1, h2")).some(node => text(node) === title);
|
|
161
|
+
if (!matchingHeading) {
|
|
162
|
+
const suffix = original.slice(original.lastIndexOf(":") + 1);
|
|
163
|
+
title = suffix.trim().split(/\s+/).length < 3 ? original.slice(original.indexOf(":") + 1) : suffix;
|
|
164
|
+
}
|
|
165
|
+
} else if (title.length > 150 || title.length < 15) {
|
|
166
|
+
const headings = elements(document.getElementsByTagName("h1"));
|
|
167
|
+
if (headings.length === 1) title = text(headings[0]);
|
|
168
|
+
}
|
|
169
|
+
title = title.trim().replace(NORMALIZE, " ");
|
|
170
|
+
if (title.split(/\s+/).length <= 4) return original;
|
|
171
|
+
return title;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
function jsonLdMetadata(document: ReadabilityDocument): Metadata {
|
|
175
|
+
for (const script of elements(document.getElementsByTagName("script"))) {
|
|
176
|
+
if (script.getAttribute("type") !== "application/ld+json") continue;
|
|
177
|
+
try {
|
|
178
|
+
const decoded: unknown = JSON.parse((script.textContent ?? "").replace(/^\s*<!\[CDATA\[|\]\]>\s*$/g, ""));
|
|
179
|
+
const records = Array.isArray(decoded) ? decoded : [decoded];
|
|
180
|
+
for (const candidate of records) {
|
|
181
|
+
if (!candidate || typeof candidate !== "object") continue;
|
|
182
|
+
const record = candidate as Record<string, unknown>;
|
|
183
|
+
if (typeof record["@type"] !== "string" || !ARTICLE_TYPES.test(record["@type"])) continue;
|
|
184
|
+
const author = record.author;
|
|
185
|
+
let byline: string | undefined;
|
|
186
|
+
if (author && typeof author === "object" && !Array.isArray(author)) {
|
|
187
|
+
const name = (author as Record<string, unknown>).name;
|
|
188
|
+
if (typeof name === "string") byline = name.trim();
|
|
189
|
+
} else if (Array.isArray(author)) {
|
|
190
|
+
const names = author.flatMap(item => {
|
|
191
|
+
if (!item || typeof item !== "object") return [];
|
|
192
|
+
const name = (item as Record<string, unknown>).name;
|
|
193
|
+
return typeof name === "string" ? [name.trim()] : [];
|
|
194
|
+
});
|
|
195
|
+
if (names.length) byline = names.join(", ");
|
|
196
|
+
}
|
|
197
|
+
const publisher = record.publisher;
|
|
198
|
+
const publisherName =
|
|
199
|
+
publisher && typeof publisher === "object" ? (publisher as Record<string, unknown>).name : undefined;
|
|
200
|
+
return {
|
|
201
|
+
title:
|
|
202
|
+
typeof record.name === "string"
|
|
203
|
+
? record.name.trim()
|
|
204
|
+
: typeof record.headline === "string"
|
|
205
|
+
? record.headline.trim()
|
|
206
|
+
: undefined,
|
|
207
|
+
byline,
|
|
208
|
+
excerpt: typeof record.description === "string" ? record.description.trim() : undefined,
|
|
209
|
+
siteName: typeof publisherName === "string" ? publisherName.trim() : undefined,
|
|
210
|
+
publishedTime: typeof record.datePublished === "string" ? record.datePublished.trim() : undefined,
|
|
211
|
+
};
|
|
212
|
+
}
|
|
213
|
+
} catch {
|
|
214
|
+
// Invalid publisher data is ignored just like malformed meta markup.
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
return {};
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
function metadataFromDocument(document: ReadabilityDocument, jsonLd: Metadata): Metadata {
|
|
221
|
+
const values = new Map<string, string>();
|
|
222
|
+
for (const meta of elements(document.getElementsByTagName("meta"))) {
|
|
223
|
+
const content = meta.getAttribute("content")?.trim();
|
|
224
|
+
if (!content) continue;
|
|
225
|
+
const property = meta.getAttribute("property")?.toLowerCase().replace(/\s/g, "");
|
|
226
|
+
const name = meta.getAttribute("name")?.toLowerCase().replace(/\s/g, "").replace(/\./g, ":");
|
|
227
|
+
if (
|
|
228
|
+
property &&
|
|
229
|
+
/^(?:article|dc|dcterm|og|twitter):(?:author|creator|description|published_time|title|site_name)$/.test(
|
|
230
|
+
property,
|
|
231
|
+
)
|
|
232
|
+
)
|
|
233
|
+
values.set(property, content);
|
|
234
|
+
else if (
|
|
235
|
+
name &&
|
|
236
|
+
/^(?:(?:dc|dcterm|og|twitter|parsely|weibo:(?:article|webpage))[-:]?)?(?:author|creator|pub-date|description|title|site_name)$/.test(
|
|
237
|
+
name,
|
|
238
|
+
)
|
|
239
|
+
)
|
|
240
|
+
values.set(name, content);
|
|
241
|
+
}
|
|
242
|
+
const articleAuthor = values.get("article:author");
|
|
243
|
+
const result: Metadata = {
|
|
244
|
+
title:
|
|
245
|
+
jsonLd.title ??
|
|
246
|
+
values.get("dc:title") ??
|
|
247
|
+
values.get("dcterm:title") ??
|
|
248
|
+
values.get("og:title") ??
|
|
249
|
+
values.get("title") ??
|
|
250
|
+
values.get("twitter:title") ??
|
|
251
|
+
titleFromDocument(document),
|
|
252
|
+
byline:
|
|
253
|
+
jsonLd.byline ??
|
|
254
|
+
values.get("dc:creator") ??
|
|
255
|
+
values.get("dcterm:creator") ??
|
|
256
|
+
values.get("author") ??
|
|
257
|
+
values.get("parsely-author") ??
|
|
258
|
+
(articleAuthor && !/^https?:\/\//.test(articleAuthor) ? articleAuthor : undefined),
|
|
259
|
+
excerpt:
|
|
260
|
+
jsonLd.excerpt ??
|
|
261
|
+
values.get("dc:description") ??
|
|
262
|
+
values.get("dcterm:description") ??
|
|
263
|
+
values.get("og:description") ??
|
|
264
|
+
values.get("description") ??
|
|
265
|
+
values.get("twitter:description"),
|
|
266
|
+
siteName: jsonLd.siteName ?? values.get("og:site_name"),
|
|
267
|
+
publishedTime:
|
|
268
|
+
jsonLd.publishedTime ?? values.get("article:published_time") ?? values.get("parsely-pub-date") ?? null,
|
|
269
|
+
};
|
|
270
|
+
return {
|
|
271
|
+
title: entityDecode(result.title) ?? undefined,
|
|
272
|
+
byline: entityDecode(result.byline) ?? undefined,
|
|
273
|
+
excerpt: entityDecode(result.excerpt) ?? undefined,
|
|
274
|
+
siteName: entityDecode(result.siteName) ?? undefined,
|
|
275
|
+
publishedTime: entityDecode(result.publishedTime),
|
|
276
|
+
};
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
/** Extracts the article body and metadata from a standards-shaped document. */
|
|
280
|
+
export class Readability<T = string> {
|
|
281
|
+
readonly #document: ReadabilityDocument;
|
|
282
|
+
readonly #options: ReadabilityOptions<T>;
|
|
283
|
+
readonly #scores = new Map<ReadabilityElement, number>();
|
|
284
|
+
#byline: string | undefined;
|
|
285
|
+
#lang: string | null = null;
|
|
286
|
+
|
|
287
|
+
constructor(document: ReadabilityDocument, options: ReadabilityOptions<T> = {}) {
|
|
288
|
+
this.#document = document;
|
|
289
|
+
this.#options = options;
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
/** Runs extraction once; the supplied document is consumed and should not be reused. */
|
|
293
|
+
parse(): ReadabilityArticle<T> | null {
|
|
294
|
+
const documentElement = this.#document.documentElement;
|
|
295
|
+
if (!documentElement) return null;
|
|
296
|
+
const max = this.#options.maxElemsToParse ?? 0;
|
|
297
|
+
if (max > 0) {
|
|
298
|
+
const count = descendants(documentElement).length + 1;
|
|
299
|
+
if (count > max) throw new Error(`Aborting parsing document; ${count} elements found`);
|
|
300
|
+
}
|
|
301
|
+
const jsonLd = this.#options.disableJSONLD ? {} : jsonLdMetadata(this.#document);
|
|
302
|
+
const metadata = metadataFromDocument(this.#document, jsonLd);
|
|
303
|
+
removeAll(this.#document, ["script", "style"]);
|
|
304
|
+
const body = this.#document.body;
|
|
305
|
+
if (!body) return null;
|
|
306
|
+
const source = body.innerHTML;
|
|
307
|
+
const attempts: Attempt[] = [];
|
|
308
|
+
for (const mode of [0, 1, 2, 3]) {
|
|
309
|
+
if (mode) body.innerHTML = source;
|
|
310
|
+
this.#scores.clear();
|
|
311
|
+
this.#byline = undefined;
|
|
312
|
+
const attempt = this.#extract(body, documentElement, metadata.title ?? "", mode);
|
|
313
|
+
if (attempt) attempts.push(attempt);
|
|
314
|
+
if (attempt && attempt.length >= (this.#options.charThreshold || 500)) break;
|
|
315
|
+
}
|
|
316
|
+
attempts.sort((left, right) => right.length - left.length);
|
|
317
|
+
const best = attempts[0];
|
|
318
|
+
if (!best?.length) return null;
|
|
319
|
+
if (!metadata.excerpt) {
|
|
320
|
+
const firstParagraph = elements(best.element.getElementsByTagName("p"))[0];
|
|
321
|
+
if (firstParagraph) metadata.excerpt = (firstParagraph.textContent ?? "").trim();
|
|
322
|
+
}
|
|
323
|
+
const contentText = best.element.textContent ?? "";
|
|
324
|
+
const serializer =
|
|
325
|
+
this.#options.serializer ?? ((node: ReadabilityNode) => (node as ReadabilityElement).innerHTML as T);
|
|
326
|
+
return {
|
|
327
|
+
title: metadata.title,
|
|
328
|
+
byline: metadata.byline ?? this.#byline,
|
|
329
|
+
dir: best.dir,
|
|
330
|
+
lang: this.#lang,
|
|
331
|
+
content: serializer(best.element),
|
|
332
|
+
textContent: contentText,
|
|
333
|
+
length: contentText.length,
|
|
334
|
+
excerpt: metadata.excerpt,
|
|
335
|
+
siteName: metadata.siteName,
|
|
336
|
+
publishedTime: metadata.publishedTime,
|
|
337
|
+
};
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
#extract(
|
|
341
|
+
body: ReadabilityElement,
|
|
342
|
+
documentElement: ReadabilityElement,
|
|
343
|
+
articleTitle: string,
|
|
344
|
+
mode: number,
|
|
345
|
+
): Attempt | null {
|
|
346
|
+
this.#lang = documentElement.getAttribute("lang");
|
|
347
|
+
const stripUnlikely = mode === 0;
|
|
348
|
+
const weightClasses = mode < 2;
|
|
349
|
+
const all = [documentElement, ...descendants(documentElement)];
|
|
350
|
+
const scored: ReadabilityElement[] = [];
|
|
351
|
+
let titleRemoved = false;
|
|
352
|
+
for (const node of all) {
|
|
353
|
+
if (node === documentElement || node.tagName === "BODY") continue;
|
|
354
|
+
const label = matchLabel(node);
|
|
355
|
+
if (!visible(node) || (node.getAttribute("aria-modal") === "true" && node.getAttribute("role") === "dialog")) {
|
|
356
|
+
node.remove();
|
|
357
|
+
continue;
|
|
358
|
+
}
|
|
359
|
+
if (!this.#byline && this.#isByline(node, label)) {
|
|
360
|
+
this.#byline = text(node);
|
|
361
|
+
node.remove();
|
|
362
|
+
continue;
|
|
363
|
+
}
|
|
364
|
+
if (!titleRemoved && /^(?:H1|H2)$/.test(node.tagName) && this.#similar(articleTitle, text(node)) > 0.75) {
|
|
365
|
+
titleRemoved = true;
|
|
366
|
+
node.remove();
|
|
367
|
+
continue;
|
|
368
|
+
}
|
|
369
|
+
if (
|
|
370
|
+
(stripUnlikely && UNLIKELY.test(label) && !POSSIBLE.test(label)) ||
|
|
371
|
+
UNLIKELY_ROLES.has(node.getAttribute("role") ?? "")
|
|
372
|
+
) {
|
|
373
|
+
node.remove();
|
|
374
|
+
continue;
|
|
375
|
+
}
|
|
376
|
+
if (SCORE_TAGS.has(node.tagName)) scored.push(node);
|
|
377
|
+
}
|
|
378
|
+
for (const paragraph of scored) this.#scoreParagraph(paragraph, weightClasses);
|
|
379
|
+
let top: ReadabilityElement | undefined;
|
|
380
|
+
let topScore = Number.NEGATIVE_INFINITY;
|
|
381
|
+
for (const [candidate, raw] of this.#scores) {
|
|
382
|
+
if (candidate.tagName === "BODY" || candidate.tagName === "HTML") continue;
|
|
383
|
+
const score = raw * (1 - linkDensity(candidate));
|
|
384
|
+
this.#scores.set(candidate, score);
|
|
385
|
+
if (score > topScore) {
|
|
386
|
+
top = candidate;
|
|
387
|
+
topScore = score;
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
if (!top || top.tagName === "BODY") top = body;
|
|
391
|
+
while (
|
|
392
|
+
top.parentNode &&
|
|
393
|
+
(top.parentNode as ReadabilityElement).tagName !== "BODY" &&
|
|
394
|
+
(top.parentNode as ReadabilityElement).children.length === 1
|
|
395
|
+
)
|
|
396
|
+
top = top.parentNode as ReadabilityElement;
|
|
397
|
+
const parent = top.parentNode as ReadabilityElement | null;
|
|
398
|
+
const article = this.#document.createElement("DIV");
|
|
399
|
+
const siblings = parent ? elements(parent.children) : [top];
|
|
400
|
+
const threshold = Math.max(10, (this.#scores.get(top) ?? topScore) * 0.2);
|
|
401
|
+
for (const sibling of siblings) {
|
|
402
|
+
const siblingText = text(sibling);
|
|
403
|
+
const sameClassBonus =
|
|
404
|
+
sibling.className && sibling.className === top.className ? (this.#scores.get(top) ?? 0) * 0.2 : 0;
|
|
405
|
+
const include =
|
|
406
|
+
sibling === top ||
|
|
407
|
+
(this.#scores.get(sibling) ?? 0) + sameClassBonus >= threshold ||
|
|
408
|
+
(sibling.tagName === "P" &&
|
|
409
|
+
((siblingText.length > 80 && linkDensity(sibling) < 0.25) ||
|
|
410
|
+
(siblingText.length > 0 &&
|
|
411
|
+
siblingText.length < 80 &&
|
|
412
|
+
linkDensity(sibling) === 0 &&
|
|
413
|
+
/\.(?: |$)/.test(siblingText))));
|
|
414
|
+
if (!include) continue;
|
|
415
|
+
if (["DIV", "ARTICLE", "SECTION", "P", "OL", "UL"].includes(sibling.tagName)) {
|
|
416
|
+
article.appendChild(sibling);
|
|
417
|
+
continue;
|
|
418
|
+
}
|
|
419
|
+
const replacement = this.#document.createElement("DIV");
|
|
420
|
+
for (const attribute of Array.from(sibling.attributes))
|
|
421
|
+
replacement.setAttribute(attribute.name, attribute.value);
|
|
422
|
+
while (sibling.firstChild) replacement.appendChild(sibling.firstChild);
|
|
423
|
+
article.appendChild(replacement);
|
|
424
|
+
}
|
|
425
|
+
this.#clean(article, mode < 3);
|
|
426
|
+
const page = this.#document.createElement("DIV");
|
|
427
|
+
page.id = "readability-page-1";
|
|
428
|
+
page.className = "page";
|
|
429
|
+
while (article.firstChild) page.appendChild(article.firstChild);
|
|
430
|
+
article.appendChild(page);
|
|
431
|
+
const content = text(article);
|
|
432
|
+
let dir: string | null | undefined;
|
|
433
|
+
let ancestor: ReadabilityNode | null = top;
|
|
434
|
+
while (ancestor) {
|
|
435
|
+
if ((ancestor as ReadabilityElement).getAttribute) {
|
|
436
|
+
dir = (ancestor as ReadabilityElement).getAttribute("dir");
|
|
437
|
+
if (dir) break;
|
|
438
|
+
}
|
|
439
|
+
ancestor = ancestor.parentNode;
|
|
440
|
+
}
|
|
441
|
+
return { element: article, length: content.length, dir };
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
#scoreParagraph(node: ReadabilityElement, weightClasses: boolean): void {
|
|
445
|
+
const content = text(node);
|
|
446
|
+
if (content.length < 25) return;
|
|
447
|
+
const score =
|
|
448
|
+
1 +
|
|
449
|
+
content.split(/[\u002c\u060c\ufe50\ufe10\ufe11\u2e41\u2e34\u2e32\uff0c]/).length +
|
|
450
|
+
Math.min(Math.floor(content.length / 100), 3);
|
|
451
|
+
let ancestor = node.parentNode;
|
|
452
|
+
for (let level = 0; ancestor && level < 5; level++, ancestor = ancestor.parentNode) {
|
|
453
|
+
const element = ancestor as ReadabilityElement;
|
|
454
|
+
if (!element.tagName || !element.parentNode || !(element.parentNode as ReadabilityElement).tagName) continue;
|
|
455
|
+
const baseline =
|
|
456
|
+
this.#scores.get(element) ?? initialScore(element) - (weightClasses ? 0 : classWeight(element));
|
|
457
|
+
const divisor = level === 0 ? 1 : level === 1 ? 2 : level * 3;
|
|
458
|
+
this.#scores.set(element, baseline + score / divisor);
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
#clean(root: ReadabilityElement, conditional: boolean): void {
|
|
463
|
+
removeAll(root, DROP_TAGS);
|
|
464
|
+
for (const heading of elements(root.querySelectorAll("h1, h2, h3, h4, h5, h6"))) {
|
|
465
|
+
if (classWeight(heading) < 0 || linkDensity(heading) > 0.33) heading.remove();
|
|
466
|
+
}
|
|
467
|
+
if (conditional) {
|
|
468
|
+
for (const node of elements(root.querySelectorAll("table, ul, div"))) {
|
|
469
|
+
if (node === root) continue;
|
|
470
|
+
const nodeText = text(node);
|
|
471
|
+
const paragraphs = node.getElementsByTagName("p").length;
|
|
472
|
+
const images = node.getElementsByTagName("img").length;
|
|
473
|
+
const inputs = node.getElementsByTagName("input").length;
|
|
474
|
+
if (
|
|
475
|
+
classWeight(node) < 0 ||
|
|
476
|
+
(!nodeText && !images) ||
|
|
477
|
+
(nodeText.split(",").length < 10 &&
|
|
478
|
+
((images > paragraphs && paragraphs > 0) ||
|
|
479
|
+
inputs > Math.floor(paragraphs / 3) ||
|
|
480
|
+
linkDensity(node) > 0.5))
|
|
481
|
+
)
|
|
482
|
+
node.remove();
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
for (const paragraph of elements(root.getElementsByTagName("p"))) {
|
|
486
|
+
if (!text(paragraph) && !paragraph.querySelector("img, embed, object, iframe")) paragraph.remove();
|
|
487
|
+
}
|
|
488
|
+
for (const node of [root, ...descendants(root)]) {
|
|
489
|
+
if (!this.#options.keepClasses) {
|
|
490
|
+
const preserved = (this.#options.classesToPreserve ?? []).filter(name =>
|
|
491
|
+
node.className.split(/\s+/).includes(name),
|
|
492
|
+
);
|
|
493
|
+
if (node.id === "readability-page-1") preserved.unshift("page");
|
|
494
|
+
if (preserved.length) node.className = [...new Set(preserved)].join(" ");
|
|
495
|
+
else node.removeAttribute("class");
|
|
496
|
+
}
|
|
497
|
+
for (const attr of [
|
|
498
|
+
"style",
|
|
499
|
+
"align",
|
|
500
|
+
"background",
|
|
501
|
+
"bgcolor",
|
|
502
|
+
"border",
|
|
503
|
+
"cellpadding",
|
|
504
|
+
"cellspacing",
|
|
505
|
+
"frame",
|
|
506
|
+
"hspace",
|
|
507
|
+
"rules",
|
|
508
|
+
"valign",
|
|
509
|
+
"vspace",
|
|
510
|
+
])
|
|
511
|
+
node.removeAttribute(attr);
|
|
512
|
+
}
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
#isByline(node: ReadabilityElement, label: string): boolean {
|
|
516
|
+
const value = text(node);
|
|
517
|
+
return (
|
|
518
|
+
value.length > 0 &&
|
|
519
|
+
value.length < 100 &&
|
|
520
|
+
(node.getAttribute("rel") === "author" ||
|
|
521
|
+
(node.getAttribute("itemprop") ?? "").includes("author") ||
|
|
522
|
+
BYLINE.test(label))
|
|
523
|
+
);
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
#similar(left: string, right: string): number {
|
|
527
|
+
const leftTokens = left.toLowerCase().split(/\W+/).filter(Boolean);
|
|
528
|
+
const rightTokens = right.toLowerCase().split(/\W+/).filter(Boolean);
|
|
529
|
+
if (!leftTokens.length || !rightTokens.length) return 0;
|
|
530
|
+
const unmatched = rightTokens.filter(token => !leftTokens.includes(token));
|
|
531
|
+
return 1 - unmatched.join(" ").length / rightTokens.join(" ").length;
|
|
532
|
+
}
|
|
533
|
+
}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/** Behavior-compatible reimplementation of @mozilla/readability's used surface. */
|
|
2
|
+
|
|
3
|
+
import type { ReadabilityDocument, ReadabilityElement, ReadabilityNode } from "./types";
|
|
4
|
+
|
|
5
|
+
const UNLIKELY =
|
|
6
|
+
/-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote/i;
|
|
7
|
+
const POSSIBLE = /and|article|body|column|content|main|shadow/i;
|
|
8
|
+
|
|
9
|
+
/** Options for the inexpensive readerability estimate. */
|
|
10
|
+
export interface ReaderableOptions {
|
|
11
|
+
minContentLength?: number;
|
|
12
|
+
minScore?: number;
|
|
13
|
+
visibilityChecker?: (node: ReadabilityNode) => boolean;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/** Estimates whether a document contains enough prose for article extraction. */
|
|
17
|
+
export function isProbablyReaderable(
|
|
18
|
+
document: ReadabilityDocument,
|
|
19
|
+
options: ReaderableOptions | ((node: ReadabilityNode) => boolean) = {},
|
|
20
|
+
): boolean {
|
|
21
|
+
const resolved = typeof options === "function" ? { visibilityChecker: options } : options;
|
|
22
|
+
const minLength = resolved.minContentLength ?? 140;
|
|
23
|
+
const minScore = resolved.minScore ?? 20;
|
|
24
|
+
const visible =
|
|
25
|
+
resolved.visibilityChecker ??
|
|
26
|
+
((node: ReadabilityNode) => {
|
|
27
|
+
const element = node as ReadabilityElement;
|
|
28
|
+
const style = element.getAttribute("style")?.toLowerCase() ?? "";
|
|
29
|
+
return (
|
|
30
|
+
!element.hasAttribute("hidden") &&
|
|
31
|
+
element.getAttribute("aria-hidden") !== "true" &&
|
|
32
|
+
!/display\s*:\s*none/.test(style)
|
|
33
|
+
);
|
|
34
|
+
});
|
|
35
|
+
const candidates = new Set<ReadabilityElement>(Array.from(document.querySelectorAll("p, pre, article")));
|
|
36
|
+
for (const br of Array.from(document.querySelectorAll("div > br"))) {
|
|
37
|
+
if (br.parentNode) candidates.add(br.parentNode as ReadabilityElement);
|
|
38
|
+
}
|
|
39
|
+
let score = 0;
|
|
40
|
+
for (const node of candidates) {
|
|
41
|
+
if (!visible(node)) continue;
|
|
42
|
+
const label = `${node.className} ${node.id}`;
|
|
43
|
+
if (UNLIKELY.test(label) && !POSSIBLE.test(label)) continue;
|
|
44
|
+
if ((node.parentNode as ReadabilityElement | null)?.tagName === "LI" && node.tagName === "P") continue;
|
|
45
|
+
const length = (node.textContent ?? "").trim().length;
|
|
46
|
+
if (length < minLength) continue;
|
|
47
|
+
score += Math.sqrt(length - minLength);
|
|
48
|
+
if (score > minScore) return true;
|
|
49
|
+
}
|
|
50
|
+
return false;
|
|
51
|
+
}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
/** Behavior-compatible reimplementation of @mozilla/readability's used surface. */
|
|
2
|
+
|
|
3
|
+
/** Minimal standards-shaped DOM node accepted by the readability extractor. */
|
|
4
|
+
export interface ReadabilityNode {
|
|
5
|
+
readonly nodeType: number;
|
|
6
|
+
readonly nodeName: string;
|
|
7
|
+
readonly tagName?: string;
|
|
8
|
+
readonly ownerDocument?: ReadabilityDocument | null;
|
|
9
|
+
parentNode: ReadabilityNode | null;
|
|
10
|
+
readonly children?: ArrayLike<ReadabilityElement>;
|
|
11
|
+
readonly childNodes: ArrayLike<ReadabilityNode>;
|
|
12
|
+
readonly firstChild: ReadabilityNode | null;
|
|
13
|
+
readonly textContent: string | null;
|
|
14
|
+
appendChild(node: ReadabilityNode): ReadabilityNode;
|
|
15
|
+
cloneNode(deep?: boolean): ReadabilityNode;
|
|
16
|
+
remove(): void;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/** Minimal standards-shaped DOM element accepted by the readability extractor. */
|
|
20
|
+
export interface ReadabilityElement extends ReadabilityNode {
|
|
21
|
+
readonly children: ArrayLike<ReadabilityElement>;
|
|
22
|
+
readonly tagName: string;
|
|
23
|
+
id: string;
|
|
24
|
+
className: string;
|
|
25
|
+
innerHTML: string;
|
|
26
|
+
readonly attributes: ArrayLike<{ readonly name: string; readonly value: string }>;
|
|
27
|
+
getAttribute(name: string): string | null;
|
|
28
|
+
hasAttribute(name: string): boolean;
|
|
29
|
+
setAttribute(name: string, value: string): void;
|
|
30
|
+
removeAttribute(name: string): void;
|
|
31
|
+
getElementsByTagName(name: string): ArrayLike<ReadabilityElement>;
|
|
32
|
+
querySelector(selector: string): ReadabilityElement | null;
|
|
33
|
+
querySelectorAll(selector: string): ArrayLike<ReadabilityElement>;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** Minimal standards-shaped document accepted by the readability extractor. */
|
|
37
|
+
export interface ReadabilityDocument extends ReadabilityNode {
|
|
38
|
+
title: string;
|
|
39
|
+
readonly body: ReadabilityElement | null;
|
|
40
|
+
readonly documentElement: ReadabilityElement | null;
|
|
41
|
+
createElement(name: string): ReadabilityElement;
|
|
42
|
+
getElementsByTagName(name: string): ArrayLike<ReadabilityElement>;
|
|
43
|
+
querySelector(selector: string): ReadabilityElement | null;
|
|
44
|
+
querySelectorAll(selector: string): ArrayLike<ReadabilityElement>;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** Configuration for article extraction. */
|
|
48
|
+
export interface ReadabilityOptions<T = string> {
|
|
49
|
+
debug?: boolean;
|
|
50
|
+
maxElemsToParse?: number;
|
|
51
|
+
nbTopCandidates?: number;
|
|
52
|
+
charThreshold?: number;
|
|
53
|
+
classesToPreserve?: string[];
|
|
54
|
+
keepClasses?: boolean;
|
|
55
|
+
serializer?: (node: ReadabilityNode) => T;
|
|
56
|
+
disableJSONLD?: boolean;
|
|
57
|
+
allowedVideoRegex?: RegExp;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** Extracted article and metadata. */
|
|
61
|
+
export interface ReadabilityArticle<T = string> {
|
|
62
|
+
title: string | null | undefined;
|
|
63
|
+
content: T | null | undefined;
|
|
64
|
+
textContent: string | null | undefined;
|
|
65
|
+
length: number | null | undefined;
|
|
66
|
+
excerpt: string | null | undefined;
|
|
67
|
+
byline: string | null | undefined;
|
|
68
|
+
dir: string | null | undefined;
|
|
69
|
+
siteName: string | null | undefined;
|
|
70
|
+
lang: string | null | undefined;
|
|
71
|
+
publishedTime: string | null | undefined;
|
|
72
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/** Behavior-compatible reimplementation of @mozilla/readability's used surface. */
|
|
2
|
+
|
|
3
|
+
export { Readability } from "./readability/readability";
|
|
4
|
+
export { isProbablyReaderable, type ReaderableOptions } from "./readability/readerable";
|
|
5
|
+
export type {
|
|
6
|
+
ReadabilityArticle,
|
|
7
|
+
ReadabilityDocument,
|
|
8
|
+
ReadabilityElement,
|
|
9
|
+
ReadabilityNode,
|
|
10
|
+
ReadabilityOptions,
|
|
11
|
+
} from "./readability/types";
|