@oh-my-pi/pi-utils 17.2.9 → 17.2.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/dist/types/acp/connection.d.ts +118 -0
- package/dist/types/acp/protocol.d.ts +526 -0
- package/dist/types/acp/schema.d.ts +41 -0
- package/dist/types/acp/stream.d.ts +8 -0
- package/dist/types/acp/transport.d.ts +81 -0
- package/dist/types/acp.d.ts +6 -0
- package/dist/types/browsers.d.ts +68 -0
- package/dist/types/chalk.d.ts +125 -0
- package/dist/types/dates.d.ts +7 -0
- package/dist/types/docx/converter.d.ts +46 -0
- package/dist/types/docx/xml.d.ts +26 -0
- package/dist/types/docx/zip.d.ts +6 -0
- package/dist/types/docx.d.ts +11 -0
- package/dist/types/dom/core.d.ts +431 -0
- package/dist/types/dom/parser.d.ts +7 -0
- package/dist/types/dom/selector.d.ts +5 -0
- package/dist/types/dom.d.ts +5 -0
- package/dist/types/headers.d.ts +34 -0
- package/dist/types/logger/rotating-file.d.ts +18 -0
- package/dist/types/lru.d.ts +46 -0
- package/dist/types/marked/core.d.ts +445 -0
- package/dist/types/marked.d.ts +2 -0
- package/dist/types/postmortem.d.ts +14 -0
- package/dist/types/procmgr.d.ts +16 -0
- package/dist/types/prompt.d.ts +2 -2
- package/dist/types/readability/readability.d.ts +9 -0
- package/dist/types/readability/readerable.d.ts +10 -0
- package/dist/types/readability/types.d.ts +70 -0
- package/dist/types/readability.d.ts +4 -0
- package/dist/types/template.d.ts +62 -0
- package/dist/types/turndown/gfm.d.ts +11 -0
- package/dist/types/turndown/html.d.ts +5 -0
- package/dist/types/turndown/service.d.ts +21 -0
- package/dist/types/turndown/types.d.ts +70 -0
- package/dist/types/turndown.d.ts +4 -0
- package/dist/types/vterm/buffer.d.ts +99 -0
- package/dist/types/vterm/terminal.d.ts +44 -0
- package/dist/types/vterm.d.ts +8 -0
- package/dist/types/xml.d.ts +31 -0
- package/package.json +2 -5
- package/src/acp/connection.ts +344 -0
- package/src/acp/protocol.ts +466 -0
- package/src/acp/schema.ts +160 -0
- package/src/acp/stream.ts +82 -0
- package/src/acp/transport.ts +213 -0
- package/src/acp.ts +6 -0
- package/src/browsers.ts +501 -0
- package/src/chalk.ts +312 -0
- package/src/dates.ts +194 -0
- package/src/docx/converter.ts +681 -0
- package/src/docx/xml.ts +166 -0
- package/src/docx/zip.ts +87 -0
- package/src/docx.ts +20 -0
- package/src/dom/core.ts +1254 -0
- package/src/dom/parser.ts +370 -0
- package/src/dom/selector.ts +290 -0
- package/src/dom.ts +33 -0
- package/src/headers.ts +167 -0
- package/src/logger/rotating-file.ts +149 -0
- package/src/logger.ts +12 -28
- package/src/lru.ts +185 -0
- package/src/marked/core.ts +1576 -0
- package/src/marked.ts +2 -0
- package/src/postmortem.ts +44 -1
- package/src/procmgr.ts +21 -4
- package/src/prompt.ts +5 -22
- package/src/readability/readability.ts +533 -0
- package/src/readability/readerable.ts +51 -0
- package/src/readability/types.ts +72 -0
- package/src/readability.ts +11 -0
- package/src/template.ts +586 -0
- package/src/turndown/gfm.ts +106 -0
- package/src/turndown/html.ts +257 -0
- package/src/turndown/service.ts +334 -0
- package/src/turndown/types.ts +81 -0
- package/src/turndown.ts +5 -0
- package/src/vterm/buffer.ts +218 -0
- package/src/vterm/terminal.ts +773 -0
- package/src/vterm.ts +8 -0
- package/src/xml.ts +313 -0
- package/src/winston-daily-rotate-file.d.ts +0 -6
package/src/marked.ts
ADDED
package/src/postmortem.ts
CHANGED
|
@@ -67,6 +67,17 @@ function exitProcess(code: number): never {
|
|
|
67
67
|
let cleanupPromise: Promise<void> | undefined;
|
|
68
68
|
let stdioDisconnectRegistrations = 0;
|
|
69
69
|
|
|
70
|
+
/** User-facing command printed before fatal cleanup so interrupted work can be resumed. */
|
|
71
|
+
export interface FatalRecoveryHint {
|
|
72
|
+
/** Stable label identifying the recoverable session or process. */
|
|
73
|
+
label: string;
|
|
74
|
+
/** Complete shell command the user can execute to resume the interrupted work. */
|
|
75
|
+
command: string;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
type FatalRecoveryHintProvider = () => FatalRecoveryHint | undefined;
|
|
79
|
+
const fatalRecoveryHintProviders = new Set<FatalRecoveryHintProvider>();
|
|
80
|
+
|
|
70
81
|
/**
|
|
71
82
|
* Internal: runs all registered cleanup callbacks for the given reason.
|
|
72
83
|
* Ensures each callback is invoked at most once. Handles errors and prevents reentrancy.
|
|
@@ -196,6 +207,38 @@ export function interceptUnhandledRejections(interceptor: (reason: unknown) => b
|
|
|
196
207
|
return () => rejectionInterceptors.delete(interceptor);
|
|
197
208
|
}
|
|
198
209
|
|
|
210
|
+
/**
|
|
211
|
+
* Register a synchronous recovery command to print when the process exits
|
|
212
|
+
* through an uncaught exception or unhandled rejection.
|
|
213
|
+
*/
|
|
214
|
+
export function registerFatalRecoveryHint(provider: FatalRecoveryHintProvider): () => void {
|
|
215
|
+
fatalRecoveryHintProviders.add(provider);
|
|
216
|
+
return () => fatalRecoveryHintProviders.delete(provider);
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
function escapeFatalHintText(value: string): string {
|
|
220
|
+
return value.replace(/[\u0000-\u001f\u007f-\u009f]/gu, char => {
|
|
221
|
+
const code = char.codePointAt(0) ?? 0;
|
|
222
|
+
return `\\u${code.toString(16).padStart(4, "0")}`;
|
|
223
|
+
});
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
function formatFatalRecoveryHints(): string {
|
|
227
|
+
const lines: string[] = [];
|
|
228
|
+
const seenCommands = new Set<string>();
|
|
229
|
+
for (const provider of fatalRecoveryHintProviders) {
|
|
230
|
+
try {
|
|
231
|
+
const hint = provider();
|
|
232
|
+
if (!hint?.command || seenCommands.has(hint.command)) continue;
|
|
233
|
+
seenCommands.add(hint.command);
|
|
234
|
+
lines.push(` ${escapeFatalHintText(hint.label)}: ${escapeFatalHintText(hint.command)}`);
|
|
235
|
+
} catch (err) {
|
|
236
|
+
logger.warn("Fatal recovery hint provider failed", { err });
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
return lines.length > 0 ? `\n[Recovery]\n${lines.join("\n")}\n` : "";
|
|
240
|
+
}
|
|
241
|
+
|
|
199
242
|
function formatFatalError(label: string, err: Error): string {
|
|
200
243
|
const name = err.name || "Error";
|
|
201
244
|
const message = err.message || "(no message)";
|
|
@@ -212,7 +255,7 @@ async function exitAfterFatal(label: string, logMessage: string, err: Error, rea
|
|
|
212
255
|
// A revoked terminal can make stream writes raise another fatal error. Use
|
|
213
256
|
// the descriptor directly so failure stays synchronous and contained.
|
|
214
257
|
try {
|
|
215
|
-
fs.writeSync(2, formatFatalError(label, err));
|
|
258
|
+
fs.writeSync(2, `${formatFatalError(label, err)}${formatFatalRecoveryHints()}`);
|
|
216
259
|
} catch {}
|
|
217
260
|
logger.error(logMessage, { err });
|
|
218
261
|
await runCleanup(reason);
|
package/src/procmgr.ts
CHANGED
|
@@ -50,12 +50,19 @@ function buildSpawnEnv(shell: string): Record<string, string> {
|
|
|
50
50
|
|
|
51
51
|
/**
|
|
52
52
|
* Get shell args for the resolved shell.
|
|
53
|
-
* cmd.exe takes `/c`;
|
|
54
|
-
*
|
|
53
|
+
* cmd.exe takes `/c`; PowerShell (powershell.exe / pwsh) takes
|
|
54
|
+
* `-NoLogo -Command`, with `-NoProfile` when PI_BASH_NO_LOGIN /
|
|
55
|
+
* CLAUDE_BASH_NO_LOGIN is set (profile scripts are PowerShell's login-shell
|
|
56
|
+
* analog); POSIX shells take `-c`, with `-l` unless the same env is set.
|
|
57
|
+
*
|
|
58
|
+
* Exported for tests; `env` overrides the process env gate.
|
|
55
59
|
*/
|
|
56
|
-
function getShellArgs(shell: string): string[] {
|
|
60
|
+
export function getShellArgs(shell: string, env: Record<string, string | undefined> = $env): string[] {
|
|
57
61
|
if (isCmdShell(shell)) return ["/c"];
|
|
58
|
-
const noLogin =
|
|
62
|
+
const noLogin = env.PI_BASH_NO_LOGIN || env.CLAUDE_BASH_NO_LOGIN;
|
|
63
|
+
if (isPowerShell(shell)) {
|
|
64
|
+
return noLogin ? ["-NoLogo", "-NoProfile", "-Command"] : ["-NoLogo", "-Command"];
|
|
65
|
+
}
|
|
59
66
|
return noLogin ? ["-c"] : ["-l", "-c"];
|
|
60
67
|
}
|
|
61
68
|
|
|
@@ -65,6 +72,16 @@ export function isCmdShell(shell: string): boolean {
|
|
|
65
72
|
return basename === "cmd.exe" || basename === "cmd";
|
|
66
73
|
}
|
|
67
74
|
|
|
75
|
+
/**
|
|
76
|
+
* Whether the shell is Windows PowerShell or PowerShell Core (pwsh). Spawn
|
|
77
|
+
* paths must use `-Command`: passing the POSIX `-l -c` pair makes PowerShell
|
|
78
|
+
* parse `-l` as the command and fail with `The term '-l' is not recognized`.
|
|
79
|
+
*/
|
|
80
|
+
export function isPowerShell(shell: string): boolean {
|
|
81
|
+
const basename = shell.replace(/\\/g, "/").split("/").pop()?.toLowerCase();
|
|
82
|
+
return basename === "powershell.exe" || basename === "powershell" || basename === "pwsh.exe" || basename === "pwsh";
|
|
83
|
+
}
|
|
84
|
+
|
|
68
85
|
/**
|
|
69
86
|
* Get shell prefix for wrapping commands (profilers, strace, etc.).
|
|
70
87
|
*/
|
package/src/prompt.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import type { HelperDelegate,
|
|
2
|
-
import Handlebars from "
|
|
1
|
+
import type { HelperDelegate, Template } from "./template";
|
|
2
|
+
import * as Handlebars from "./template";
|
|
3
3
|
|
|
4
|
-
export type { HelperDelegate, HelperOptions, Template, TemplateDelegate };
|
|
4
|
+
export type { HelperDelegate, HelperOptions, Template, TemplateDelegate } from "./template";
|
|
5
5
|
|
|
6
6
|
export type PromptRenderPhase = "pre-render" | "post-render";
|
|
7
7
|
|
|
@@ -525,30 +525,13 @@ export function registerPartial(name: string, fn: Template): void {
|
|
|
525
525
|
handlebars.registerPartial(name, fn);
|
|
526
526
|
}
|
|
527
527
|
|
|
528
|
-
/**
|
|
529
|
-
* Handlebars' lexer greedily matches `}}}` as `CLOSE_UNESCAPED` (the close of a
|
|
530
|
-
* triple-stash `{{{ ... }}}`). When a regular helper close `}}` is immediately
|
|
531
|
-
* followed by a literal `}` (common in compact JSON examples like
|
|
532
|
-
* `{del:{{href ...}}}`), the lexer mistakes the trailing `}}}` for a triple-close
|
|
533
|
-
* and rejects the input.
|
|
534
|
-
*
|
|
535
|
-
* We never use triple-stash (it's redundant under `noEscape: true`), so any run
|
|
536
|
-
* of 3+ closing braces is unambiguously "helper close `}}`" + "literal `}`s".
|
|
537
|
-
* Inject a no-op comment between them so the lexer tokenizes the helper close
|
|
538
|
-
* cleanly and treats the rest as content.
|
|
539
|
-
*/
|
|
540
|
-
function disambiguateClosingBraces(template: string): string {
|
|
541
|
-
return template.replace(/\}\}(\}+)/g, "}}{{!---}}$1");
|
|
542
|
-
}
|
|
543
|
-
|
|
544
528
|
const compiledTemplateCache = new Map<string, (context: TemplateContext) => string>();
|
|
545
529
|
|
|
546
530
|
export function compile(template: string): (context: TemplateContext) => string {
|
|
547
|
-
// Keyed on the raw template so repeat renders skip
|
|
548
|
-
// (a full-template regex pass) as well as the Handlebars compile.
|
|
531
|
+
// Keyed on the raw template so repeat renders skip parsing.
|
|
549
532
|
const cached = compiledTemplateCache.get(template);
|
|
550
533
|
if (cached) return cached;
|
|
551
|
-
const compiled = handlebars.compile(
|
|
534
|
+
const compiled = handlebars.compile(template, { noEscape: true, strict: false }) as (
|
|
552
535
|
context: TemplateContext,
|
|
553
536
|
) => string;
|
|
554
537
|
compiledTemplateCache.set(template, compiled);
|
|
@@ -0,0 +1,533 @@
|
|
|
1
|
+
/** Behavior-compatible reimplementation of @mozilla/readability's used surface. */
|
|
2
|
+
|
|
3
|
+
import type {
|
|
4
|
+
ReadabilityArticle,
|
|
5
|
+
ReadabilityDocument,
|
|
6
|
+
ReadabilityElement,
|
|
7
|
+
ReadabilityNode,
|
|
8
|
+
ReadabilityOptions,
|
|
9
|
+
} from "./types";
|
|
10
|
+
|
|
11
|
+
const UNLIKELY =
|
|
12
|
+
/-ad-|ai2html|banner|breadcrumbs|comment|community|combx|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup/i;
|
|
13
|
+
const POSSIBLE = /and|article|body|column|content|main|shadow/i;
|
|
14
|
+
const POSITIVE = /article|body|content|entry|hentry|h-entry|main|page|pagination|post|text|blog|story/i;
|
|
15
|
+
const NEGATIVE =
|
|
16
|
+
/-ad-|hidden|^hid$| hid$| hid |^hid |banner|comment|com-|contact|footer|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|widget/i;
|
|
17
|
+
const BYLINE = /byline|author|dateline|writtenby|p-author/i;
|
|
18
|
+
const SCORE_TAGS = new Set(["SECTION", "H2", "H3", "H4", "H5", "H6", "P", "TD", "PRE"]);
|
|
19
|
+
const DROP_TAGS = [
|
|
20
|
+
"form",
|
|
21
|
+
"fieldset",
|
|
22
|
+
"object",
|
|
23
|
+
"embed",
|
|
24
|
+
"footer",
|
|
25
|
+
"link",
|
|
26
|
+
"aside",
|
|
27
|
+
"iframe",
|
|
28
|
+
"input",
|
|
29
|
+
"textarea",
|
|
30
|
+
"select",
|
|
31
|
+
"button",
|
|
32
|
+
];
|
|
33
|
+
const UNLIKELY_ROLES = new Set(["menu", "menubar", "complementary", "navigation", "alert", "alertdialog", "dialog"]);
|
|
34
|
+
const ARTICLE_TYPES =
|
|
35
|
+
/^(?:Article|AdvertiserContentArticle|NewsArticle|AnalysisNewsArticle|OpinionNewsArticle|ReportageNewsArticle|ReviewNewsArticle|Report|ScholarlyArticle|MedicalScholarlyArticle|SocialMediaPosting|BlogPosting|LiveBlogPosting|DiscussionForumPosting|TechArticle|APIReference)$/;
|
|
36
|
+
const NORMALIZE = /\s{2,}/g;
|
|
37
|
+
|
|
38
|
+
type Metadata = {
|
|
39
|
+
title?: string;
|
|
40
|
+
byline?: string;
|
|
41
|
+
excerpt?: string;
|
|
42
|
+
siteName?: string;
|
|
43
|
+
publishedTime?: string | null;
|
|
44
|
+
};
|
|
45
|
+
|
|
46
|
+
type Attempt = { element: ReadabilityElement; length: number; dir?: string | null };
|
|
47
|
+
|
|
48
|
+
function elements(collection: ArrayLike<ReadabilityElement>): ReadabilityElement[] {
|
|
49
|
+
return Array.from(collection);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function descendants(root: ReadabilityElement): ReadabilityElement[] {
|
|
53
|
+
const result: ReadabilityElement[] = [];
|
|
54
|
+
const pending = elements(root.children).reverse();
|
|
55
|
+
while (pending.length) {
|
|
56
|
+
const node = pending.pop();
|
|
57
|
+
if (!node) continue;
|
|
58
|
+
result.push(node);
|
|
59
|
+
const children = elements(node.children);
|
|
60
|
+
for (let index = children.length - 1; index >= 0; index--) pending.push(children[index]);
|
|
61
|
+
}
|
|
62
|
+
return result;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function text(node: ReadabilityNode): string {
|
|
66
|
+
return (node.textContent ?? "").trim().replace(NORMALIZE, " ");
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function matchLabel(node: ReadabilityElement): string {
|
|
70
|
+
return `${typeof node.className === "string" ? node.className : ""} ${node.id ?? ""}`;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function classWeight(node: ReadabilityElement): number {
|
|
74
|
+
const label = matchLabel(node);
|
|
75
|
+
return (POSITIVE.test(label) ? 25 : 0) - (NEGATIVE.test(label) ? 25 : 0);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function initialScore(node: ReadabilityElement): number {
|
|
79
|
+
let score = classWeight(node);
|
|
80
|
+
switch (node.tagName) {
|
|
81
|
+
case "DIV":
|
|
82
|
+
score += 5;
|
|
83
|
+
break;
|
|
84
|
+
case "PRE":
|
|
85
|
+
case "TD":
|
|
86
|
+
case "BLOCKQUOTE":
|
|
87
|
+
score += 3;
|
|
88
|
+
break;
|
|
89
|
+
case "ADDRESS":
|
|
90
|
+
case "OL":
|
|
91
|
+
case "UL":
|
|
92
|
+
case "DL":
|
|
93
|
+
case "DD":
|
|
94
|
+
case "DT":
|
|
95
|
+
case "LI":
|
|
96
|
+
case "FORM":
|
|
97
|
+
score -= 3;
|
|
98
|
+
break;
|
|
99
|
+
case "H1":
|
|
100
|
+
case "H2":
|
|
101
|
+
case "H3":
|
|
102
|
+
case "H4":
|
|
103
|
+
case "H5":
|
|
104
|
+
case "H6":
|
|
105
|
+
case "TH":
|
|
106
|
+
score -= 5;
|
|
107
|
+
break;
|
|
108
|
+
}
|
|
109
|
+
return score;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
function linkDensity(node: ReadabilityElement): number {
|
|
113
|
+
const total = text(node).length;
|
|
114
|
+
if (!total) return 0;
|
|
115
|
+
let linked = 0;
|
|
116
|
+
for (const link of elements(node.getElementsByTagName("a"))) {
|
|
117
|
+
linked += text(link).length * ((link.getAttribute("href") ?? "").startsWith("#") ? 0.3 : 1);
|
|
118
|
+
}
|
|
119
|
+
return linked / total;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
function visible(node: ReadabilityElement): boolean {
|
|
123
|
+
const style = node.getAttribute("style")?.toLowerCase() ?? "";
|
|
124
|
+
return (
|
|
125
|
+
!node.hasAttribute("hidden") &&
|
|
126
|
+
node.getAttribute("aria-hidden") !== "true" &&
|
|
127
|
+
!/display\s*:\s*none|visibility\s*:\s*hidden/.test(style)
|
|
128
|
+
);
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
function removeAll(root: ReadabilityNode, tags: readonly string[]): void {
|
|
132
|
+
const container = root as ReadabilityElement;
|
|
133
|
+
for (const tag of tags) {
|
|
134
|
+
for (const node of elements(container.getElementsByTagName(tag))) node.remove();
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
function entityDecode(value: string | undefined | null): string | undefined | null {
|
|
139
|
+
if (!value) return value;
|
|
140
|
+
const named: Record<string, string> = { quot: '"', amp: "&", apos: "'", lt: "<", gt: ">" };
|
|
141
|
+
return value
|
|
142
|
+
.replace(/&(quot|amp|apos|lt|gt);/g, (_, name: string) => named[name] ?? "")
|
|
143
|
+
.replace(/&#(?:x([0-9a-f]+)|(\d+));/gi, (_, hex: string | undefined, decimal: string | undefined) => {
|
|
144
|
+
const value = Number.parseInt(hex ?? decimal ?? "0", hex ? 16 : 10);
|
|
145
|
+
return String.fromCodePoint(
|
|
146
|
+
value === 0 || value > 0x10ffff || (value >= 0xd800 && value <= 0xdfff) ? 0xfffd : value,
|
|
147
|
+
);
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
function titleFromDocument(document: ReadabilityDocument): string {
|
|
152
|
+
const titleElement = elements(document.getElementsByTagName("title"))[0];
|
|
153
|
+
const original = typeof document.title === "string" ? document.title.trim() : titleElement ? text(titleElement) : "";
|
|
154
|
+
let title = original;
|
|
155
|
+
const separators = [...original.matchAll(/ [|\\/>»-] /g)];
|
|
156
|
+
if (separators.length) {
|
|
157
|
+
title = original.slice(0, separators.at(-1)?.index);
|
|
158
|
+
if (title.trim().split(/\s+/).length < 3) title = original.replace(/^[^|\\/>»-]*[|\\/>»-]/, "");
|
|
159
|
+
} else if (title.includes(": ")) {
|
|
160
|
+
const matchingHeading = elements(document.querySelectorAll("h1, h2")).some(node => text(node) === title);
|
|
161
|
+
if (!matchingHeading) {
|
|
162
|
+
const suffix = original.slice(original.lastIndexOf(":") + 1);
|
|
163
|
+
title = suffix.trim().split(/\s+/).length < 3 ? original.slice(original.indexOf(":") + 1) : suffix;
|
|
164
|
+
}
|
|
165
|
+
} else if (title.length > 150 || title.length < 15) {
|
|
166
|
+
const headings = elements(document.getElementsByTagName("h1"));
|
|
167
|
+
if (headings.length === 1) title = text(headings[0]);
|
|
168
|
+
}
|
|
169
|
+
title = title.trim().replace(NORMALIZE, " ");
|
|
170
|
+
if (title.split(/\s+/).length <= 4) return original;
|
|
171
|
+
return title;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
function jsonLdMetadata(document: ReadabilityDocument): Metadata {
|
|
175
|
+
for (const script of elements(document.getElementsByTagName("script"))) {
|
|
176
|
+
if (script.getAttribute("type") !== "application/ld+json") continue;
|
|
177
|
+
try {
|
|
178
|
+
const decoded: unknown = JSON.parse((script.textContent ?? "").replace(/^\s*<!\[CDATA\[|\]\]>\s*$/g, ""));
|
|
179
|
+
const records = Array.isArray(decoded) ? decoded : [decoded];
|
|
180
|
+
for (const candidate of records) {
|
|
181
|
+
if (!candidate || typeof candidate !== "object") continue;
|
|
182
|
+
const record = candidate as Record<string, unknown>;
|
|
183
|
+
if (typeof record["@type"] !== "string" || !ARTICLE_TYPES.test(record["@type"])) continue;
|
|
184
|
+
const author = record.author;
|
|
185
|
+
let byline: string | undefined;
|
|
186
|
+
if (author && typeof author === "object" && !Array.isArray(author)) {
|
|
187
|
+
const name = (author as Record<string, unknown>).name;
|
|
188
|
+
if (typeof name === "string") byline = name.trim();
|
|
189
|
+
} else if (Array.isArray(author)) {
|
|
190
|
+
const names = author.flatMap(item => {
|
|
191
|
+
if (!item || typeof item !== "object") return [];
|
|
192
|
+
const name = (item as Record<string, unknown>).name;
|
|
193
|
+
return typeof name === "string" ? [name.trim()] : [];
|
|
194
|
+
});
|
|
195
|
+
if (names.length) byline = names.join(", ");
|
|
196
|
+
}
|
|
197
|
+
const publisher = record.publisher;
|
|
198
|
+
const publisherName =
|
|
199
|
+
publisher && typeof publisher === "object" ? (publisher as Record<string, unknown>).name : undefined;
|
|
200
|
+
return {
|
|
201
|
+
title:
|
|
202
|
+
typeof record.name === "string"
|
|
203
|
+
? record.name.trim()
|
|
204
|
+
: typeof record.headline === "string"
|
|
205
|
+
? record.headline.trim()
|
|
206
|
+
: undefined,
|
|
207
|
+
byline,
|
|
208
|
+
excerpt: typeof record.description === "string" ? record.description.trim() : undefined,
|
|
209
|
+
siteName: typeof publisherName === "string" ? publisherName.trim() : undefined,
|
|
210
|
+
publishedTime: typeof record.datePublished === "string" ? record.datePublished.trim() : undefined,
|
|
211
|
+
};
|
|
212
|
+
}
|
|
213
|
+
} catch {
|
|
214
|
+
// Invalid publisher data is ignored just like malformed meta markup.
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
return {};
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
function metadataFromDocument(document: ReadabilityDocument, jsonLd: Metadata): Metadata {
|
|
221
|
+
const values = new Map<string, string>();
|
|
222
|
+
for (const meta of elements(document.getElementsByTagName("meta"))) {
|
|
223
|
+
const content = meta.getAttribute("content")?.trim();
|
|
224
|
+
if (!content) continue;
|
|
225
|
+
const property = meta.getAttribute("property")?.toLowerCase().replace(/\s/g, "");
|
|
226
|
+
const name = meta.getAttribute("name")?.toLowerCase().replace(/\s/g, "").replace(/\./g, ":");
|
|
227
|
+
if (
|
|
228
|
+
property &&
|
|
229
|
+
/^(?:article|dc|dcterm|og|twitter):(?:author|creator|description|published_time|title|site_name)$/.test(
|
|
230
|
+
property,
|
|
231
|
+
)
|
|
232
|
+
)
|
|
233
|
+
values.set(property, content);
|
|
234
|
+
else if (
|
|
235
|
+
name &&
|
|
236
|
+
/^(?:(?:dc|dcterm|og|twitter|parsely|weibo:(?:article|webpage))[-:]?)?(?:author|creator|pub-date|description|title|site_name)$/.test(
|
|
237
|
+
name,
|
|
238
|
+
)
|
|
239
|
+
)
|
|
240
|
+
values.set(name, content);
|
|
241
|
+
}
|
|
242
|
+
const articleAuthor = values.get("article:author");
|
|
243
|
+
const result: Metadata = {
|
|
244
|
+
title:
|
|
245
|
+
jsonLd.title ??
|
|
246
|
+
values.get("dc:title") ??
|
|
247
|
+
values.get("dcterm:title") ??
|
|
248
|
+
values.get("og:title") ??
|
|
249
|
+
values.get("title") ??
|
|
250
|
+
values.get("twitter:title") ??
|
|
251
|
+
titleFromDocument(document),
|
|
252
|
+
byline:
|
|
253
|
+
jsonLd.byline ??
|
|
254
|
+
values.get("dc:creator") ??
|
|
255
|
+
values.get("dcterm:creator") ??
|
|
256
|
+
values.get("author") ??
|
|
257
|
+
values.get("parsely-author") ??
|
|
258
|
+
(articleAuthor && !/^https?:\/\//.test(articleAuthor) ? articleAuthor : undefined),
|
|
259
|
+
excerpt:
|
|
260
|
+
jsonLd.excerpt ??
|
|
261
|
+
values.get("dc:description") ??
|
|
262
|
+
values.get("dcterm:description") ??
|
|
263
|
+
values.get("og:description") ??
|
|
264
|
+
values.get("description") ??
|
|
265
|
+
values.get("twitter:description"),
|
|
266
|
+
siteName: jsonLd.siteName ?? values.get("og:site_name"),
|
|
267
|
+
publishedTime:
|
|
268
|
+
jsonLd.publishedTime ?? values.get("article:published_time") ?? values.get("parsely-pub-date") ?? null,
|
|
269
|
+
};
|
|
270
|
+
return {
|
|
271
|
+
title: entityDecode(result.title) ?? undefined,
|
|
272
|
+
byline: entityDecode(result.byline) ?? undefined,
|
|
273
|
+
excerpt: entityDecode(result.excerpt) ?? undefined,
|
|
274
|
+
siteName: entityDecode(result.siteName) ?? undefined,
|
|
275
|
+
publishedTime: entityDecode(result.publishedTime),
|
|
276
|
+
};
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
/** Extracts the article body and metadata from a standards-shaped document. */
|
|
280
|
+
export class Readability<T = string> {
|
|
281
|
+
readonly #document: ReadabilityDocument;
|
|
282
|
+
readonly #options: ReadabilityOptions<T>;
|
|
283
|
+
readonly #scores = new Map<ReadabilityElement, number>();
|
|
284
|
+
#byline: string | undefined;
|
|
285
|
+
#lang: string | null = null;
|
|
286
|
+
|
|
287
|
+
constructor(document: ReadabilityDocument, options: ReadabilityOptions<T> = {}) {
|
|
288
|
+
this.#document = document;
|
|
289
|
+
this.#options = options;
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
/** Runs extraction once; the supplied document is consumed and should not be reused. */
|
|
293
|
+
parse(): ReadabilityArticle<T> | null {
|
|
294
|
+
const documentElement = this.#document.documentElement;
|
|
295
|
+
if (!documentElement) return null;
|
|
296
|
+
const max = this.#options.maxElemsToParse ?? 0;
|
|
297
|
+
if (max > 0) {
|
|
298
|
+
const count = descendants(documentElement).length + 1;
|
|
299
|
+
if (count > max) throw new Error(`Aborting parsing document; ${count} elements found`);
|
|
300
|
+
}
|
|
301
|
+
const jsonLd = this.#options.disableJSONLD ? {} : jsonLdMetadata(this.#document);
|
|
302
|
+
const metadata = metadataFromDocument(this.#document, jsonLd);
|
|
303
|
+
removeAll(this.#document, ["script", "style"]);
|
|
304
|
+
const body = this.#document.body;
|
|
305
|
+
if (!body) return null;
|
|
306
|
+
const source = body.innerHTML;
|
|
307
|
+
const attempts: Attempt[] = [];
|
|
308
|
+
for (const mode of [0, 1, 2, 3]) {
|
|
309
|
+
if (mode) body.innerHTML = source;
|
|
310
|
+
this.#scores.clear();
|
|
311
|
+
this.#byline = undefined;
|
|
312
|
+
const attempt = this.#extract(body, documentElement, metadata.title ?? "", mode);
|
|
313
|
+
if (attempt) attempts.push(attempt);
|
|
314
|
+
if (attempt && attempt.length >= (this.#options.charThreshold || 500)) break;
|
|
315
|
+
}
|
|
316
|
+
attempts.sort((left, right) => right.length - left.length);
|
|
317
|
+
const best = attempts[0];
|
|
318
|
+
if (!best?.length) return null;
|
|
319
|
+
if (!metadata.excerpt) {
|
|
320
|
+
const firstParagraph = elements(best.element.getElementsByTagName("p"))[0];
|
|
321
|
+
if (firstParagraph) metadata.excerpt = (firstParagraph.textContent ?? "").trim();
|
|
322
|
+
}
|
|
323
|
+
const contentText = best.element.textContent ?? "";
|
|
324
|
+
const serializer =
|
|
325
|
+
this.#options.serializer ?? ((node: ReadabilityNode) => (node as ReadabilityElement).innerHTML as T);
|
|
326
|
+
return {
|
|
327
|
+
title: metadata.title,
|
|
328
|
+
byline: metadata.byline ?? this.#byline,
|
|
329
|
+
dir: best.dir,
|
|
330
|
+
lang: this.#lang,
|
|
331
|
+
content: serializer(best.element),
|
|
332
|
+
textContent: contentText,
|
|
333
|
+
length: contentText.length,
|
|
334
|
+
excerpt: metadata.excerpt,
|
|
335
|
+
siteName: metadata.siteName,
|
|
336
|
+
publishedTime: metadata.publishedTime,
|
|
337
|
+
};
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
#extract(
|
|
341
|
+
body: ReadabilityElement,
|
|
342
|
+
documentElement: ReadabilityElement,
|
|
343
|
+
articleTitle: string,
|
|
344
|
+
mode: number,
|
|
345
|
+
): Attempt | null {
|
|
346
|
+
this.#lang = documentElement.getAttribute("lang");
|
|
347
|
+
const stripUnlikely = mode === 0;
|
|
348
|
+
const weightClasses = mode < 2;
|
|
349
|
+
const all = [documentElement, ...descendants(documentElement)];
|
|
350
|
+
const scored: ReadabilityElement[] = [];
|
|
351
|
+
let titleRemoved = false;
|
|
352
|
+
for (const node of all) {
|
|
353
|
+
if (node === documentElement || node.tagName === "BODY") continue;
|
|
354
|
+
const label = matchLabel(node);
|
|
355
|
+
if (!visible(node) || (node.getAttribute("aria-modal") === "true" && node.getAttribute("role") === "dialog")) {
|
|
356
|
+
node.remove();
|
|
357
|
+
continue;
|
|
358
|
+
}
|
|
359
|
+
if (!this.#byline && this.#isByline(node, label)) {
|
|
360
|
+
this.#byline = text(node);
|
|
361
|
+
node.remove();
|
|
362
|
+
continue;
|
|
363
|
+
}
|
|
364
|
+
if (!titleRemoved && /^(?:H1|H2)$/.test(node.tagName) && this.#similar(articleTitle, text(node)) > 0.75) {
|
|
365
|
+
titleRemoved = true;
|
|
366
|
+
node.remove();
|
|
367
|
+
continue;
|
|
368
|
+
}
|
|
369
|
+
if (
|
|
370
|
+
(stripUnlikely && UNLIKELY.test(label) && !POSSIBLE.test(label)) ||
|
|
371
|
+
UNLIKELY_ROLES.has(node.getAttribute("role") ?? "")
|
|
372
|
+
) {
|
|
373
|
+
node.remove();
|
|
374
|
+
continue;
|
|
375
|
+
}
|
|
376
|
+
if (SCORE_TAGS.has(node.tagName)) scored.push(node);
|
|
377
|
+
}
|
|
378
|
+
for (const paragraph of scored) this.#scoreParagraph(paragraph, weightClasses);
|
|
379
|
+
let top: ReadabilityElement | undefined;
|
|
380
|
+
let topScore = Number.NEGATIVE_INFINITY;
|
|
381
|
+
for (const [candidate, raw] of this.#scores) {
|
|
382
|
+
if (candidate.tagName === "BODY" || candidate.tagName === "HTML") continue;
|
|
383
|
+
const score = raw * (1 - linkDensity(candidate));
|
|
384
|
+
this.#scores.set(candidate, score);
|
|
385
|
+
if (score > topScore) {
|
|
386
|
+
top = candidate;
|
|
387
|
+
topScore = score;
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
if (!top || top.tagName === "BODY") top = body;
|
|
391
|
+
while (
|
|
392
|
+
top.parentNode &&
|
|
393
|
+
(top.parentNode as ReadabilityElement).tagName !== "BODY" &&
|
|
394
|
+
(top.parentNode as ReadabilityElement).children.length === 1
|
|
395
|
+
)
|
|
396
|
+
top = top.parentNode as ReadabilityElement;
|
|
397
|
+
const parent = top.parentNode as ReadabilityElement | null;
|
|
398
|
+
const article = this.#document.createElement("DIV");
|
|
399
|
+
const siblings = parent ? elements(parent.children) : [top];
|
|
400
|
+
const threshold = Math.max(10, (this.#scores.get(top) ?? topScore) * 0.2);
|
|
401
|
+
for (const sibling of siblings) {
|
|
402
|
+
const siblingText = text(sibling);
|
|
403
|
+
const sameClassBonus =
|
|
404
|
+
sibling.className && sibling.className === top.className ? (this.#scores.get(top) ?? 0) * 0.2 : 0;
|
|
405
|
+
const include =
|
|
406
|
+
sibling === top ||
|
|
407
|
+
(this.#scores.get(sibling) ?? 0) + sameClassBonus >= threshold ||
|
|
408
|
+
(sibling.tagName === "P" &&
|
|
409
|
+
((siblingText.length > 80 && linkDensity(sibling) < 0.25) ||
|
|
410
|
+
(siblingText.length > 0 &&
|
|
411
|
+
siblingText.length < 80 &&
|
|
412
|
+
linkDensity(sibling) === 0 &&
|
|
413
|
+
/\.(?: |$)/.test(siblingText))));
|
|
414
|
+
if (!include) continue;
|
|
415
|
+
if (["DIV", "ARTICLE", "SECTION", "P", "OL", "UL"].includes(sibling.tagName)) {
|
|
416
|
+
article.appendChild(sibling);
|
|
417
|
+
continue;
|
|
418
|
+
}
|
|
419
|
+
const replacement = this.#document.createElement("DIV");
|
|
420
|
+
for (const attribute of Array.from(sibling.attributes))
|
|
421
|
+
replacement.setAttribute(attribute.name, attribute.value);
|
|
422
|
+
while (sibling.firstChild) replacement.appendChild(sibling.firstChild);
|
|
423
|
+
article.appendChild(replacement);
|
|
424
|
+
}
|
|
425
|
+
this.#clean(article, mode < 3);
|
|
426
|
+
const page = this.#document.createElement("DIV");
|
|
427
|
+
page.id = "readability-page-1";
|
|
428
|
+
page.className = "page";
|
|
429
|
+
while (article.firstChild) page.appendChild(article.firstChild);
|
|
430
|
+
article.appendChild(page);
|
|
431
|
+
const content = text(article);
|
|
432
|
+
let dir: string | null | undefined;
|
|
433
|
+
let ancestor: ReadabilityNode | null = top;
|
|
434
|
+
while (ancestor) {
|
|
435
|
+
if ((ancestor as ReadabilityElement).getAttribute) {
|
|
436
|
+
dir = (ancestor as ReadabilityElement).getAttribute("dir");
|
|
437
|
+
if (dir) break;
|
|
438
|
+
}
|
|
439
|
+
ancestor = ancestor.parentNode;
|
|
440
|
+
}
|
|
441
|
+
return { element: article, length: content.length, dir };
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
#scoreParagraph(node: ReadabilityElement, weightClasses: boolean): void {
|
|
445
|
+
const content = text(node);
|
|
446
|
+
if (content.length < 25) return;
|
|
447
|
+
const score =
|
|
448
|
+
1 +
|
|
449
|
+
content.split(/[\u002c\u060c\ufe50\ufe10\ufe11\u2e41\u2e34\u2e32\uff0c]/).length +
|
|
450
|
+
Math.min(Math.floor(content.length / 100), 3);
|
|
451
|
+
let ancestor = node.parentNode;
|
|
452
|
+
for (let level = 0; ancestor && level < 5; level++, ancestor = ancestor.parentNode) {
|
|
453
|
+
const element = ancestor as ReadabilityElement;
|
|
454
|
+
if (!element.tagName || !element.parentNode || !(element.parentNode as ReadabilityElement).tagName) continue;
|
|
455
|
+
const baseline =
|
|
456
|
+
this.#scores.get(element) ?? initialScore(element) - (weightClasses ? 0 : classWeight(element));
|
|
457
|
+
const divisor = level === 0 ? 1 : level === 1 ? 2 : level * 3;
|
|
458
|
+
this.#scores.set(element, baseline + score / divisor);
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
#clean(root: ReadabilityElement, conditional: boolean): void {
|
|
463
|
+
removeAll(root, DROP_TAGS);
|
|
464
|
+
for (const heading of elements(root.querySelectorAll("h1, h2, h3, h4, h5, h6"))) {
|
|
465
|
+
if (classWeight(heading) < 0 || linkDensity(heading) > 0.33) heading.remove();
|
|
466
|
+
}
|
|
467
|
+
if (conditional) {
|
|
468
|
+
for (const node of elements(root.querySelectorAll("table, ul, div"))) {
|
|
469
|
+
if (node === root) continue;
|
|
470
|
+
const nodeText = text(node);
|
|
471
|
+
const paragraphs = node.getElementsByTagName("p").length;
|
|
472
|
+
const images = node.getElementsByTagName("img").length;
|
|
473
|
+
const inputs = node.getElementsByTagName("input").length;
|
|
474
|
+
if (
|
|
475
|
+
classWeight(node) < 0 ||
|
|
476
|
+
(!nodeText && !images) ||
|
|
477
|
+
(nodeText.split(",").length < 10 &&
|
|
478
|
+
((images > paragraphs && paragraphs > 0) ||
|
|
479
|
+
inputs > Math.floor(paragraphs / 3) ||
|
|
480
|
+
linkDensity(node) > 0.5))
|
|
481
|
+
)
|
|
482
|
+
node.remove();
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
for (const paragraph of elements(root.getElementsByTagName("p"))) {
|
|
486
|
+
if (!text(paragraph) && !paragraph.querySelector("img, embed, object, iframe")) paragraph.remove();
|
|
487
|
+
}
|
|
488
|
+
for (const node of [root, ...descendants(root)]) {
|
|
489
|
+
if (!this.#options.keepClasses) {
|
|
490
|
+
const preserved = (this.#options.classesToPreserve ?? []).filter(name =>
|
|
491
|
+
node.className.split(/\s+/).includes(name),
|
|
492
|
+
);
|
|
493
|
+
if (node.id === "readability-page-1") preserved.unshift("page");
|
|
494
|
+
if (preserved.length) node.className = [...new Set(preserved)].join(" ");
|
|
495
|
+
else node.removeAttribute("class");
|
|
496
|
+
}
|
|
497
|
+
for (const attr of [
|
|
498
|
+
"style",
|
|
499
|
+
"align",
|
|
500
|
+
"background",
|
|
501
|
+
"bgcolor",
|
|
502
|
+
"border",
|
|
503
|
+
"cellpadding",
|
|
504
|
+
"cellspacing",
|
|
505
|
+
"frame",
|
|
506
|
+
"hspace",
|
|
507
|
+
"rules",
|
|
508
|
+
"valign",
|
|
509
|
+
"vspace",
|
|
510
|
+
])
|
|
511
|
+
node.removeAttribute(attr);
|
|
512
|
+
}
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
#isByline(node: ReadabilityElement, label: string): boolean {
|
|
516
|
+
const value = text(node);
|
|
517
|
+
return (
|
|
518
|
+
value.length > 0 &&
|
|
519
|
+
value.length < 100 &&
|
|
520
|
+
(node.getAttribute("rel") === "author" ||
|
|
521
|
+
(node.getAttribute("itemprop") ?? "").includes("author") ||
|
|
522
|
+
BYLINE.test(label))
|
|
523
|
+
);
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
#similar(left: string, right: string): number {
|
|
527
|
+
const leftTokens = left.toLowerCase().split(/\W+/).filter(Boolean);
|
|
528
|
+
const rightTokens = right.toLowerCase().split(/\W+/).filter(Boolean);
|
|
529
|
+
if (!leftTokens.length || !rightTokens.length) return 0;
|
|
530
|
+
const unmatched = rightTokens.filter(token => !leftTokens.includes(token));
|
|
531
|
+
return 1 - unmatched.join(" ").length / rightTokens.join(" ").length;
|
|
532
|
+
}
|
|
533
|
+
}
|