dsh-web-fetch-playwright 0.2.18 → 0.2.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -1
- package/README.zh-CN.md +17 -1
- package/lib/client.js +18 -18
- package/lib/index.js +58203 -423
- package/lib/types/block-model.d.ts +2 -0
- package/lib/types/blocks.d.ts +67 -0
- package/lib/types/candidates.d.ts +157 -0
- package/lib/types/index.d.ts +6 -2
- package/lib/types/markdown.d.ts +50 -24
- package/lib/types/pdf.d.ts +143 -0
- package/lib/types/provider.d.ts +9 -0
- package/lib/types/selector-model.d.ts +2 -0
- package/lib/types/selector.d.ts +58 -0
- package/lib/types/trees.d.ts +25 -0
- package/lib/types/types.d.ts +5 -0
- package/package.json +3 -2
- package/src/block-model.ts +5 -0
- package/src/blocks.ts +223 -0
- package/src/candidates.ts +639 -0
- package/src/index.ts +6 -2
- package/src/markdown.ts +136 -413
- package/src/pdf.ts +1114 -0
- package/src/provider.ts +117 -8
- package/src/selector-model.ts +6 -0
- package/src/selector.ts +277 -0
- package/src/session-window.ts +8 -0
- package/src/trees.ts +43 -0
- package/src/types.ts +5 -0
package/src/markdown.ts
CHANGED
|
@@ -1,21 +1,19 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* The denoise pipeline: rendered HTML → sanitized article HTML → markdown.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
* inline `data:` image payloads are elided to size placeholders
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
* options and span-safe table rules as the shipped `dsh-tool-web` renderer,
|
|
18
|
-
* so output is consistent with what `web_fetch` produces elsewhere.
|
|
4
|
+
* Since 0.2.20 the pipeline is "propose, then select": jsdom parses the
|
|
5
|
+
* page; inline `data:` image payloads are elided to size placeholders;
|
|
6
|
+
* every extraction heuristic proposes a candidate (`./candidates.ts` —
|
|
7
|
+
* Readability pruned/unpruned/lenient, the semantic main container, the
|
|
8
|
+
* whole document with and without consent/chrome stripping, the repeated
|
|
9
|
+
* record region, noscript-unwrapped variants, the Discourse preloaded post
|
|
10
|
+
* stream) plus the block classifier's pick (`./blocks.ts`); DOMPurify
|
|
11
|
+
* sanitizes each and forbids the layout tags noise lives in; a learned
|
|
12
|
+
* selector (`./selector.ts`) estimates each candidate's quality from page-
|
|
13
|
+
* and candidate-level features and the best one wins, with the 0.2.18
|
|
14
|
+
* choice kept unless another candidate clearly beats it. Turndown with the
|
|
15
|
+
* GFM plugin converts the winner using the same style options and span-safe
|
|
16
|
+
* table rules as the shipped `dsh-tool-web` renderer.
|
|
19
17
|
*
|
|
20
18
|
* Synchronous for its callers and unit-tested against fixture pages: jsdom
|
|
21
19
|
* itself is loaded lazily on the first call — and, inside a dsh profile, only
|
|
@@ -27,10 +25,12 @@
|
|
|
27
25
|
|
|
28
26
|
import { createRequire } from 'node:module'
|
|
29
27
|
import createDOMPurify from 'dompurify'
|
|
30
|
-
import { Readability } from '@mozilla/readability'
|
|
31
28
|
import TurndownService from 'turndown'
|
|
32
29
|
import { gfm } from '@joplin/turndown-plugin-gfm'
|
|
33
30
|
import { installBuiltinSlashResolveCompat } from './host-resolve-compat.ts'
|
|
31
|
+
import { FORBID_TAGS, generateCandidates, type CandidateKind, type RawCandidate, visibleTextLength } from './candidates.ts'
|
|
32
|
+
import { candidateFeatures, type CandidateStats, measureCandidate, pageFeatures, scoreCandidate } from './selector.ts'
|
|
33
|
+
import { type Block, blockFeatures, blockKey, blockThreshold, classifyBlocks, hasBlockModel, renderBlocks, segmentBlocks } from './blocks.ts'
|
|
34
34
|
|
|
35
35
|
/** `createRequire` anchored at this module, so jsdom resolves from this install. */
|
|
36
36
|
const require_ = createRequire(import.meta.url)
|
|
@@ -58,13 +58,6 @@ function loadJsdom(): JsdomModule {
|
|
|
58
58
|
return jsdom
|
|
59
59
|
}
|
|
60
60
|
|
|
61
|
-
/** Layout/noise tags DOMPurify removes outright (with their content). */
|
|
62
|
-
const FORBID_TAGS = [
|
|
63
|
-
'nav', 'aside', 'header', 'footer', 'form', 'svg', 'iframe', 'noscript',
|
|
64
|
-
'button', 'select', 'option', 'input', 'textarea', 'dialog', 'canvas',
|
|
65
|
-
'video', 'audio', 'template',
|
|
66
|
-
]
|
|
67
|
-
|
|
68
61
|
/** Attributes stripped from sanitized output (styling survives as noise). */
|
|
69
62
|
const FORBID_ATTR = ['style', 'class', 'id', 'hidden', 'aria-hidden', 'role']
|
|
70
63
|
|
|
@@ -114,105 +107,6 @@ function humanSize(bytes: number): string {
|
|
|
114
107
|
return `${(bytes / 1_048_576).toFixed(1)}MB`
|
|
115
108
|
}
|
|
116
109
|
|
|
117
|
-
/**
|
|
118
|
-
* Tags whose subtrees are noise when measuring how much of a page an
|
|
119
|
-
* extraction kept: DOMPurify drops these outright (`KEEP_CONTENT: false`),
|
|
120
|
-
* and its default policy always removes `script`/`style` — so a text walk
|
|
121
|
-
* that skips them measures what the sanitized document fallback could
|
|
122
|
-
* actually return, not text that is already destined for deletion. Measuring
|
|
123
|
-
* the raw body instead would let a huge `<nav>`/`<footer>` mass shrink an
|
|
124
|
-
* article's honest coverage share and demote good extractions.
|
|
125
|
-
*/
|
|
126
|
-
const NOISE_SUBTREE_TAGS = new Set([...FORBID_TAGS, 'script', 'style'])
|
|
127
|
-
|
|
128
|
-
/**
|
|
129
|
-
* A page must have at least this much visible (post-noise) text before the
|
|
130
|
-
* article acceptance gate below has an opinion. Sparse pages — logins,
|
|
131
|
-
* stubs, `login-like` fixtures — have nowhere for content to hide, so
|
|
132
|
-
* Readability's pick, however short, is still the best reading of the page
|
|
133
|
-
* and keeps flowing through the article path exactly as before.
|
|
134
|
-
*/
|
|
135
|
-
const GATE_MIN_BODY_TEXT_CHARS = 1_000
|
|
136
|
-
|
|
137
|
-
/**
|
|
138
|
-
* Minimum non-whitespace characters a Readability extraction must carry
|
|
139
|
-
* before it can be trusted as a page's content. Measured on the 163.com
|
|
140
|
-
* homepage, the scorer's pick was a 96-character footer licensing block —
|
|
141
|
-
* real articles and even sparse pages clear two-to-three times this bar,
|
|
142
|
-
* while link-dense portal pages lose whole news columns to it.
|
|
143
|
-
*/
|
|
144
|
-
const GATE_MIN_ARTICLE_TEXT_CHARS = 250
|
|
145
|
-
|
|
146
|
-
/**
|
|
147
|
-
* Minimum share of a page's visible text an extraction must cover to be
|
|
148
|
-
* trusted. Measured separation on real pages is wide: portal homepages
|
|
149
|
-
* (163 0.01, sina 0.03, sohu 0.02) vs article pages (0.28–0.45), so 10%
|
|
150
|
-
* sits in an empty band with margin on both sides.
|
|
151
|
-
*/
|
|
152
|
-
const GATE_MIN_ARTICLE_COVERAGE = 0.1
|
|
153
|
-
|
|
154
|
-
/**
|
|
155
|
-
* Count non-whitespace characters under `root`, skipping noise subtrees
|
|
156
|
-
* ({@link NOISE_SUBTREE_TAGS}). Whitespace is ignored so CJK pages (no
|
|
157
|
-
* spaces) and latin pages (space-separated) measure comparable densities,
|
|
158
|
-
* mirroring how the extraction-quality ratios in the docs were computed.
|
|
159
|
-
*
|
|
160
|
-
* @param root - the subtree to measure; `null` counts as nothing.
|
|
161
|
-
* @returns the number of non-whitespace characters in kept subtrees.
|
|
162
|
-
*/
|
|
163
|
-
function visibleTextLength(root: Node | null): number {
|
|
164
|
-
if (root === null) return 0
|
|
165
|
-
let length = 0
|
|
166
|
-
const visit = (node: Node): void => {
|
|
167
|
-
// Text nodes contribute their non-whitespace characters (nodeType 3).
|
|
168
|
-
if (node.nodeType === 3) {
|
|
169
|
-
length += (node.nodeValue ?? '').replace(/\s+/g, '').length
|
|
170
|
-
return
|
|
171
|
-
}
|
|
172
|
-
// Only elements branch further (nodeType 1); comments and friends carry
|
|
173
|
-
// no text, and noise subtrees are skipped wholesale.
|
|
174
|
-
if (node.nodeType !== 1 || NOISE_SUBTREE_TAGS.has(node.nodeName.toLowerCase())) return
|
|
175
|
-
for (let child = node.firstChild; child !== null; child = child.nextSibling) visit(child)
|
|
176
|
-
}
|
|
177
|
-
visit(root)
|
|
178
|
-
return length
|
|
179
|
-
}
|
|
180
|
-
|
|
181
|
-
/**
|
|
182
|
-
* Whether a Readability extraction is trustworthy as *the* content of the
|
|
183
|
-
* page it came from — the acceptance gate that keeps list-shaped pages whole.
|
|
184
|
-
*
|
|
185
|
-
* Readability is an article extractor: it scores text mass and penalizes
|
|
186
|
-
* link density, so on a portal/catalog page — where the content *is* the
|
|
187
|
-
* links — every news column scores near zero, some sparse low-link block (a
|
|
188
|
-
* footer licensing notice, a province-switcher popup) wins by default, and
|
|
189
|
-
* Readability's own 500-char quality check cannot fail the parse because its
|
|
190
|
-
* retry loop eventually returns the longest of its bad attempts. The plugin
|
|
191
|
-
* used to accept any non-null result, so the 163.com homepage rendered as a
|
|
192
|
-
* 162-character licensing fragment while 320 headlines were dropped — a
|
|
193
|
-
* recall loss far worse than any precision the gate could cost.
|
|
194
|
-
*
|
|
195
|
-
* The gate therefore distrusts an extraction that is tiny in absolute terms
|
|
196
|
-
* or covers a sliver of the page's visible text; the pipeline then falls
|
|
197
|
-
* back to the sanitized whole document, which DOMPurify keeps clean and
|
|
198
|
-
* which on such pages retains effectively all list content (measured 0%
|
|
199
|
-
* text loss on the 163.com homepage). Only text-rich pages are gated
|
|
200
|
-
* ({@link GATE_MIN_BODY_TEXT_CHARS}); below that, behavior is unchanged.
|
|
201
|
-
*
|
|
202
|
-
* @param articleHtml - the `article.content` HTML Readability produced.
|
|
203
|
-
* @param document - the live document the article was extracted from.
|
|
204
|
-
* @returns whether the article should flow through the article path.
|
|
205
|
-
*/
|
|
206
|
-
function articleCoversPage(articleHtml: string, document: Document): boolean {
|
|
207
|
-
const bodyText = visibleTextLength(document.body)
|
|
208
|
-
if (bodyText < GATE_MIN_BODY_TEXT_CHARS) return true
|
|
209
|
-
const container = document.createElement('div')
|
|
210
|
-
container.innerHTML = articleHtml
|
|
211
|
-
const articleText = visibleTextLength(container)
|
|
212
|
-
return articleText >= GATE_MIN_ARTICLE_TEXT_CHARS
|
|
213
|
-
&& articleText / bodyText >= GATE_MIN_ARTICLE_COVERAGE
|
|
214
|
-
}
|
|
215
|
-
|
|
216
110
|
/** The shared converter: same style options as `dsh-tool-web`'s renderer. */
|
|
217
111
|
const turndown = new TurndownService({
|
|
218
112
|
headingStyle: 'atx',
|
|
@@ -276,329 +170,158 @@ export interface DenoiseResult {
|
|
|
276
170
|
/** The markdown body (title, when found, already prepended). */
|
|
277
171
|
markdown: string
|
|
278
172
|
/**
|
|
279
|
-
* `article` =
|
|
280
|
-
* `document` = whole-document
|
|
281
|
-
*
|
|
173
|
+
* `article` = a content-focused candidate (Readability, main container,
|
|
174
|
+
* record region, forum post stream); `document` = a whole-document
|
|
175
|
+
* candidate (sanitized body, optionally chrome-stripped).
|
|
282
176
|
*/
|
|
283
177
|
mode: DenoiseMode
|
|
178
|
+
/** The candidate strategy the selector picked. */
|
|
179
|
+
strategy: CandidateKind
|
|
284
180
|
}
|
|
285
181
|
|
|
286
|
-
/**
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
if (root === null) return 0
|
|
296
|
-
let length = 0
|
|
297
|
-
const visit = (node: Node, insideAnchor: boolean): void => {
|
|
298
|
-
if (node.nodeType === 3) {
|
|
299
|
-
if (insideAnchor) length += (node.nodeValue ?? '').replace(/\s+/g, '').length
|
|
300
|
-
return
|
|
301
|
-
}
|
|
302
|
-
if (node.nodeType !== 1 || NOISE_SUBTREE_TAGS.has(node.nodeName.toLowerCase())) return
|
|
303
|
-
const anchor = insideAnchor || node.nodeName.toLowerCase() === 'a'
|
|
304
|
-
for (let child = node.firstChild; child !== null; child = child.nextSibling) visit(child, anchor)
|
|
305
|
-
}
|
|
306
|
-
visit(root, false)
|
|
307
|
-
return length
|
|
308
|
-
}
|
|
309
|
-
|
|
310
|
-
/**
|
|
311
|
-
* Minimum non-whitespace characters a block must carry before its link
|
|
312
|
-
* density is even evaluated. Blocks below this are spacers, empty wrappers,
|
|
313
|
-
* or one-line sign-offs — too small to be worth a removal decision, so the
|
|
314
|
-
* trailing walk skips past them instead of stopping at them.
|
|
315
|
-
*/
|
|
316
|
-
const TRAILING_MIN_TEXT_CHARS = 40
|
|
317
|
-
|
|
318
|
-
/**
|
|
319
|
-
* Link-density share above which a trailing block reads as a link farm:
|
|
320
|
-
* recommendation feeds, related-article rails, tag clouds, share bars.
|
|
321
|
-
* Measured on the 163.com dy-article badcase the section's pick carried 84
|
|
322
|
-
* linked characters inside 183 total (0.46) — recommendation cards dilute
|
|
323
|
-
* their density with author/date lines, so a 0.5 bar would miss them by
|
|
324
|
-
* construction; an honest closing paragraph with a citation or two measures
|
|
325
|
-
* 0.1–0.3, leaving 0.4 in the empty band between.
|
|
326
|
-
*/
|
|
327
|
-
const TRAILING_LINK_DENSITY = 0.4
|
|
328
|
-
|
|
329
|
-
/**
|
|
330
|
-
* A trailing block qualifies as a link farm only when it is link-heavy in
|
|
331
|
-
* aggregate too: at least {@link TRAILING_LINK_COUNT} anchors or
|
|
332
|
-
* {@link TRAILING_LINK_TEXT_CHARS} linked characters. Density alone would
|
|
333
|
-
* let a single long linked sentence (an inline citation) drag a legitimate
|
|
334
|
-
* paragraph under the knife.
|
|
335
|
-
*/
|
|
336
|
-
const TRAILING_LINK_COUNT = 3
|
|
337
|
-
const TRAILING_LINK_TEXT_CHARS = 60
|
|
338
|
-
|
|
339
|
-
/**
|
|
340
|
-
* A trailing block may never hold more than this share of the whole
|
|
341
|
-
* article's text. The prune targets *appended* widgets; a block that IS most
|
|
342
|
-
* of the extraction says the scorer picked the widget, which is the coverage
|
|
343
|
-
* gate's failure mode to handle, not this walk's.
|
|
344
|
-
*/
|
|
345
|
-
const TRAILING_MAX_ARTICLE_SHARE = 0.6
|
|
346
|
-
|
|
347
|
-
/**
|
|
348
|
-
* A trailing run may never hold more than this share of its container's
|
|
349
|
-
* text — the aggregate form of the guard above, and the one that keeps
|
|
350
|
-
* list-shaped extractions whole. A recommendation rail is a *minority*
|
|
351
|
-
* appended to real content (the 163 rail is 44% of its container); a
|
|
352
|
-
* portal or feed whose extraction IS a list of link rows (measured on a
|
|
353
|
-
* Hacker News homepage capture: ~91 trailing rows, every one individually
|
|
354
|
-
* link-dense at 0.69–0.94) has a tail run holding ~97% of the container —
|
|
355
|
-
* there the tail IS the content, the walk declines wholesale, and only a
|
|
356
|
-
* run that is a minority of its container is removed.
|
|
357
|
-
*/
|
|
358
|
-
const TRAILING_MAX_RUN_SHARE = 0.6
|
|
359
|
-
|
|
360
|
-
/**
|
|
361
|
-
* Link-density band within which a run and the block the walk stopped at
|
|
362
|
-
* read as the same material. When the content immediately before the run
|
|
363
|
-
* is itself as link-dense as the run — measured on a Hacker News homepage
|
|
364
|
-
* capture, the rows after the run's cut measure 0.94 against the run's
|
|
365
|
-
* 0.91 — the container is a *uniform list* (feed rows, portal headlines)
|
|
366
|
-
* and the tail is that list's own content, not a widget appended after it;
|
|
367
|
-
* a widget hangs after genuinely different material (the 163 rail's 0.44
|
|
368
|
-
* follows body text at 0.00). Only a run denser than what precedes it is
|
|
369
|
-
* removed.
|
|
370
|
-
*/
|
|
371
|
-
const TRAILING_UNIFORM_DENSITY_BAND = 0.25
|
|
372
|
-
|
|
373
|
-
/**
|
|
374
|
-
* Prune trailing link-dense sections from an accepted Readability
|
|
375
|
-
* extraction — the post-article widget class the scorer itself cannot be
|
|
376
|
-
* taught to drop.
|
|
377
|
-
*
|
|
378
|
-
* Why this exists (badcase: `https://www.163.com/dy/article/…`): Chinese
|
|
379
|
-
* portal article pages append a recommendation feed inside the very wrapper
|
|
380
|
-
* Readability scores as the article (`#content` holds the body, the
|
|
381
|
-
* platform disclaimer, then a rail of image/title/author/date cards). Every
|
|
382
|
-
* existing noise layer misses it: Readability's `unlikelyCandidates` /
|
|
383
|
-
* `negative` vocabularies know `related` but not `recommend` (nor the
|
|
384
|
-
* pinyin families `tuijian`/`xiangguan`), and its own cleaning pass draws
|
|
385
|
-
* the "mostly links" line at per-block density 0.5 — these cards measure
|
|
386
|
-
* 0.42–0.48 (the title links over title+author+date text), so every card
|
|
387
|
-
* clears it; in the extracted HTML the rail is a *bare* div — class and id
|
|
388
|
-
* are gone — so no class-based rule can ever see it again; DOMPurify's
|
|
389
|
-
* FORBID_TAGS works on tags, and the rail is plain div/p/h3/a; and the
|
|
390
|
-
* coverage gate cannot help because the noise rides *inside* the
|
|
391
|
-
* extraction it measures (the badcase passed at 0.103 — the rail's text
|
|
392
|
-
* was in both numerator and denominator). Modern sites with CSS-module
|
|
393
|
-
* class names (`css-1b3x7r`) present the same shape with no vocabulary to
|
|
394
|
-
* match at all, so the only signal that survives all of this is
|
|
395
|
-
* structural: a *trailing* run of blocks whose text is mostly links.
|
|
396
|
-
*
|
|
397
|
-
* For every container in the extraction the walk gathers the maximal
|
|
398
|
-
* trailing run of qualifying children — link-farm blocks per the constants
|
|
399
|
-
* above, plus sub-threshold spacers (empty wrapper divs) — and removes the
|
|
400
|
-
* run only when BOTH structural guards say "appended widget, not content":
|
|
401
|
-
* the run is a minority of the container's text
|
|
402
|
-
* ({@link TRAILING_MAX_RUN_SHARE}), and the run is clearly denser than the
|
|
403
|
-
* block the walk stopped at ({@link TRAILING_UNIFORM_DENSITY_BAND}). The
|
|
404
|
-
* guards split the two worlds the per-block view cannot: a rail is
|
|
405
|
-
* one-or-few appended blocks after genuinely different material (the 163
|
|
406
|
-
* rail's 0.44 density follows body text at 0.00), while a link-list
|
|
407
|
-
* extraction — feed rows, portal headlines — is *uniform*: its rows before
|
|
408
|
-
* any cut measure the same density as the rows after it (0.91 vs 0.94 on
|
|
409
|
-
* the Hacker News capture), so no tail of it can be mistaken for a widget.
|
|
410
|
-
*
|
|
411
|
-
* Deliberately NOT applied to the whole-document fallback: that path
|
|
412
|
-
* exists for list-shaped pages where the links *are* the content (the
|
|
413
|
-
* 163.com homepage fix), and pruning link-dense regions there would
|
|
414
|
-
* re-introduce exactly the recall loss the coverage gate exists to
|
|
415
|
-
* prevent.
|
|
416
|
-
*
|
|
417
|
-
* @param container - parsed `article.content`; mutated in place.
|
|
418
|
-
*/
|
|
419
|
-
function pruneTrailingLinkDenseSections(container: Element): void {
|
|
420
|
-
const articleText = visibleTextLength(container)
|
|
421
|
-
if (articleText === 0) return
|
|
422
|
-
// The container itself leads the walk — Readability usually wraps its
|
|
423
|
-
// content in div#readability-page-1, but an unwrapped extraction's
|
|
424
|
-
// top-level blocks are the container's direct children and must be
|
|
425
|
-
// examined too. querySelectorAll snapshots before any removal; entries
|
|
426
|
-
// inside an already-removed subtree are skipped by the contains() check,
|
|
427
|
-
// and parents are visited before children, so a deeper run inside a block
|
|
428
|
-
// an outer run is taking never competes with it.
|
|
429
|
-
for (const element of [container, ...Array.from(container.querySelectorAll('*'))]) {
|
|
430
|
-
if (!container.contains(element) && element !== container) continue
|
|
431
|
-
removeTrailingLinkDenseRun(element, articleText)
|
|
432
|
-
}
|
|
182
|
+
/** One candidate after sanitization, with its selector inputs. */
|
|
183
|
+
export interface ScoredCandidate {
|
|
184
|
+
kind: CandidateKind
|
|
185
|
+
title?: string
|
|
186
|
+
legacy: boolean
|
|
187
|
+
root: HTMLElement
|
|
188
|
+
stats: CandidateStats
|
|
189
|
+
vector: number[]
|
|
190
|
+
score: number
|
|
433
191
|
}
|
|
434
192
|
|
|
435
|
-
|
|
436
|
-
* Remove one container's maximal trailing run of link-dense children when
|
|
437
|
-
* that run is a minority of the container's text AND clearly different
|
|
438
|
-
* material from the content that precedes it.
|
|
439
|
-
*
|
|
440
|
-
* @param parent - the container whose element children are examined from
|
|
441
|
-
* the end.
|
|
442
|
-
* @param articleText - the whole article's text, for the per-block share
|
|
443
|
-
* guard.
|
|
444
|
-
*/
|
|
445
|
-
function removeTrailingLinkDenseRun(parent: Element, articleText: number): void {
|
|
446
|
-
const children = Array.from(parent.children)
|
|
447
|
-
if (children.length === 0) return
|
|
448
|
-
const containerText = visibleTextLength(parent)
|
|
449
|
-
if (containerText === 0) return
|
|
450
|
-
const run: Element[] = []
|
|
451
|
-
let runText = 0
|
|
452
|
-
let runLinked = 0
|
|
453
|
-
let stopIndex = -1
|
|
454
|
-
for (let index = children.length - 1; index >= 0; index--) {
|
|
455
|
-
const child = children[index]
|
|
456
|
-
if (child === undefined) break
|
|
457
|
-
const text = visibleTextLength(child)
|
|
458
|
-
// Too small to judge (spacer, empty wrapper): it belongs to the run
|
|
459
|
-
// but cannot extend it — keep walking toward earlier siblings.
|
|
460
|
-
if (text < TRAILING_MIN_TEXT_CHARS) {
|
|
461
|
-
run.push(child)
|
|
462
|
-
continue
|
|
463
|
-
}
|
|
464
|
-
const linked = linkTextLength(child)
|
|
465
|
-
const linkCount = child.querySelectorAll('a').length
|
|
466
|
-
if (
|
|
467
|
-
linked / text >= TRAILING_LINK_DENSITY
|
|
468
|
-
&& (linkCount >= TRAILING_LINK_COUNT || linked >= TRAILING_LINK_TEXT_CHARS)
|
|
469
|
-
&& text <= TRAILING_MAX_ARTICLE_SHARE * articleText
|
|
470
|
-
) {
|
|
471
|
-
run.push(child)
|
|
472
|
-
runText += text
|
|
473
|
-
runLinked += linked
|
|
474
|
-
continue
|
|
475
|
-
}
|
|
476
|
-
// Real content that is not a link farm: everything before it is
|
|
477
|
-
// anchored by actual article flow — stop and compare against it.
|
|
478
|
-
stopIndex = index
|
|
479
|
-
break
|
|
480
|
-
}
|
|
481
|
-
if (runText === 0 || stopIndex === -1) {
|
|
482
|
-
// Nothing to remove — or the run reached the top of the container (no
|
|
483
|
-
// preceding content to differ from): a tail that IS the whole
|
|
484
|
-
// container is list content, never a widget.
|
|
485
|
-
return
|
|
486
|
-
}
|
|
487
|
-
// The tail is the same material as what precedes it (a uniform list's
|
|
488
|
-
// rows): decline wholesale.
|
|
489
|
-
const stopChild = children[stopIndex]
|
|
490
|
-
if (stopChild === undefined) return
|
|
491
|
-
const stopDensity = linkTextLength(stopChild) / Math.max(1, visibleTextLength(stopChild))
|
|
492
|
-
if (Math.abs(runLinked / runText - stopDensity) <= TRAILING_UNIFORM_DENSITY_BAND) return
|
|
493
|
-
// The tail holds most of the container (a link-list page's rows): decline
|
|
494
|
-
// wholesale.
|
|
495
|
-
if (runText > TRAILING_MAX_RUN_SHARE * containerText) return
|
|
496
|
-
for (const child of run) child.remove()
|
|
497
|
-
}
|
|
193
|
+
const DOCUMENT_KINDS = new Set<CandidateKind>(['doc', 'docc', 'ns_doc'])
|
|
498
194
|
|
|
499
|
-
/**
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
* prune, auto-trusted by the gate) keeps its pruned form — its trailing
|
|
506
|
-
* links were noise all the same.
|
|
507
|
-
*
|
|
508
|
-
* @param articleHtml - the `article.content` HTML Readability produced.
|
|
509
|
-
* @param document - the live document, for parsing the fragment.
|
|
510
|
-
* @returns the pruned article HTML, or the original on a misfire.
|
|
511
|
-
*/
|
|
512
|
-
function prunedArticleContent(articleHtml: string, document: Document): string {
|
|
513
|
-
const container = document.createElement('div')
|
|
514
|
-
container.innerHTML = articleHtml
|
|
515
|
-
const before = visibleTextLength(container)
|
|
516
|
-
if (before === 0) return articleHtml
|
|
517
|
-
pruneTrailingLinkDenseSections(container)
|
|
518
|
-
const after = visibleTextLength(container)
|
|
519
|
-
if (after === 0 || (before >= GATE_MIN_ARTICLE_TEXT_CHARS && after < GATE_MIN_ARTICLE_TEXT_CHARS)) {
|
|
520
|
-
return articleHtml
|
|
521
|
-
}
|
|
522
|
-
return container.innerHTML
|
|
195
|
+
/** Everything one parse produced, before selection. */
|
|
196
|
+
interface Analysis {
|
|
197
|
+
sanitized: { kind: CandidateKind, title?: string, legacy: boolean, root: HTMLElement, stats: CandidateStats }[]
|
|
198
|
+
page: number[]
|
|
199
|
+
blocks: Block[]
|
|
200
|
+
blockVectors: number[][]
|
|
523
201
|
}
|
|
524
202
|
|
|
525
203
|
/**
|
|
526
|
-
*
|
|
527
|
-
*
|
|
528
|
-
* Readability failure (non-article pages, or extractions the coverage gate
|
|
529
|
-
* {@link articleCoversPage} distrusts) degrades to converting the sanitized
|
|
530
|
-
* whole document — layout tags are still forbidden, so the fallback stays
|
|
531
|
-
* cleaner than raw turndown, and a degraded page beats an error for a body
|
|
532
|
-
* the browser already rendered.
|
|
533
|
-
*
|
|
534
|
-
* @param html - the rendered page HTML (`page.content()`).
|
|
535
|
-
* @param url - the page URL, used to resolve relative links during parsing.
|
|
536
|
-
* @returns the markdown and the extraction mode used.
|
|
204
|
+
* Parse, generate candidates, sanitize and measure them, compute the page
|
|
205
|
+
* features, and featurize the page's text blocks for the block classifier.
|
|
537
206
|
*/
|
|
538
|
-
|
|
207
|
+
function analyze(html: string, url: string): Analysis {
|
|
539
208
|
const { JSDOM, VirtualConsole } = loadJsdom()
|
|
540
209
|
// jsdom's virtual console defaults to forwarding parse noise; a silent one
|
|
541
210
|
// keeps broken inline CSS on random pages out of host logs.
|
|
542
211
|
const dom = new JSDOM(html, { url, virtualConsole: new VirtualConsole() })
|
|
543
212
|
const purify = createDOMPurify(dom.window)
|
|
544
213
|
const document = dom.window.document
|
|
545
|
-
|
|
546
|
-
// Before anything downstream reads the DOM: shrink inline data-URI image
|
|
547
|
-
// payloads to size placeholders. Mutating the live document here covers
|
|
548
|
-
// both paths — the Readability clone below, and the document.body fallback
|
|
549
|
-
// (innerHTML reflects the rewritten src) — while Turndown's default image
|
|
550
|
-
// rule keeps handling alt/title escaping on the already-short src.
|
|
214
|
+
// Before anything reads the DOM: shrink inline data-URI image payloads.
|
|
551
215
|
elideDataUriImages(document)
|
|
552
216
|
|
|
553
|
-
let
|
|
554
|
-
let title: string | undefined
|
|
217
|
+
let generated: ReturnType<typeof generateCandidates>
|
|
555
218
|
try {
|
|
556
|
-
|
|
557
|
-
// rejects sparse-but-real articles (measured on a browser-rendered fixture),
|
|
558
|
-
// so the extraction is simply attempted; a null, empty, or gate-rejected
|
|
559
|
-
// result falls back to the sanitized whole document below. parse() mutates,
|
|
560
|
-
// hence the clone (typed as Document: DOM lib types cloneNode's return as
|
|
561
|
-
// Node).
|
|
562
|
-
const cloned = document.cloneNode(true) as typeof document
|
|
563
|
-
const article = new Readability(cloned).parse()
|
|
564
|
-
if (
|
|
565
|
-
article !== null && typeof article.content === 'string' && article.content !== ''
|
|
566
|
-
&& articleCoversPage(article.content, document)
|
|
567
|
-
) {
|
|
568
|
-
// Gate first, prune second, and in that order on purpose: the gate
|
|
569
|
-
// must judge Readability's pick as delivered (the widget rides inside
|
|
570
|
-
// it), and the prune then strips the post-article link farms the
|
|
571
|
-
// scorer could not be taught — see pruneTrailingLinkDenseSections.
|
|
572
|
-
source = prunedArticleContent(article.content, document)
|
|
573
|
-
title = article.title ?? undefined
|
|
574
|
-
}
|
|
219
|
+
generated = generateCandidates(document)
|
|
575
220
|
} catch {
|
|
576
|
-
//
|
|
577
|
-
|
|
221
|
+
// Pathological DOMs: the whole document is still usable.
|
|
222
|
+
generated = {
|
|
223
|
+
candidates: [{ kind: 'doc', html: document.body?.innerHTML ?? '', legacy: true }],
|
|
224
|
+
facts: { readabilityNull: true, gatePass: false, pruneChanged: false, noscriptText: 0, record: null, hasMain: false, hasPreloaded: false, main: null },
|
|
225
|
+
}
|
|
578
226
|
}
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
227
|
+
const sanitize = (candidate: RawCandidate): Analysis['sanitized'][number] => {
|
|
228
|
+
// KEEP_CONTENT: false is the load-bearing half of the denoise: DOMPurify
|
|
229
|
+
// un-wraps forbidden tags but keeps their text by default; with it, the
|
|
230
|
+
// whole subtree of a forbidden element goes.
|
|
231
|
+
const root = purify.sanitize(candidate.html, { FORBID_TAGS, FORBID_ATTR, KEEP_CONTENT: false, RETURN_DOM: true }) as HTMLElement
|
|
232
|
+
return { kind: candidate.kind, title: candidate.title, legacy: candidate.legacy, root, stats: measureCandidate(candidate.kind, candidate.legacy, root) }
|
|
233
|
+
}
|
|
234
|
+
const sanitized = generated.candidates.map(sanitize)
|
|
235
|
+
const page = pageFeatures(document, url, generated.facts, sanitized.find(c => c.kind === 'doc')?.stats, visibleTextLength(document.body))
|
|
236
|
+
|
|
237
|
+
let blocks: Block[] = []
|
|
238
|
+
let blockVectors: number[][] = []
|
|
239
|
+
if (document.body !== null) {
|
|
240
|
+
try {
|
|
241
|
+
blocks = segmentBlocks(document.body, generated.facts.main, new Set(generated.facts.record?.items ?? []))
|
|
242
|
+
.filter(block => block.words > 0)
|
|
243
|
+
const art = sanitized.find(c => c.kind === 'art')
|
|
244
|
+
const artKeys = new Set(art === undefined ? [] : segmentBlocks(art.root).map(block => blockKey(block.text)))
|
|
245
|
+
blockVectors = blockFeatures(blocks, artKeys).map(vector => [...page, ...vector])
|
|
246
|
+
} catch {
|
|
247
|
+
blocks = []
|
|
248
|
+
blockVectors = []
|
|
249
|
+
}
|
|
583
250
|
}
|
|
251
|
+
if (blocks.length > 0 && hasBlockModel()) {
|
|
252
|
+
const threshold = blockThreshold()
|
|
253
|
+
const keep = classifyBlocks(blockVectors).map(p => p >= threshold)
|
|
254
|
+
if (keep.some(Boolean)) {
|
|
255
|
+
const title = sanitized.find(c => c.title !== undefined)?.title
|
|
256
|
+
sanitized.push(sanitize({ kind: 'blk', html: renderBlocks(blocks, keep), title, legacy: false }))
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
return { sanitized, page, blocks, blockVectors }
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
/**
|
|
263
|
+
* Parse, generate, sanitize, featurize, and score every candidate.
|
|
264
|
+
* @returns candidates sorted by the selector's estimate, best first.
|
|
265
|
+
*/
|
|
266
|
+
export function scoreCandidates(html: string, url: string): ScoredCandidate[] {
|
|
267
|
+
const { sanitized, page } = analyze(html, url)
|
|
268
|
+
const allStats = sanitized.map(candidate => candidate.stats)
|
|
269
|
+
const scored = sanitized.map(candidate => {
|
|
270
|
+
const vector = [...page, ...candidateFeatures(candidate.stats, allStats)]
|
|
271
|
+
return { ...candidate, vector, score: scoreCandidate(vector) }
|
|
272
|
+
})
|
|
273
|
+
// Stable: ties keep generation order (legacy-first preference).
|
|
274
|
+
return scored.sort((a, b) => b.score - a.score)
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
/**
|
|
278
|
+
* Block-classifier training rows for one page (benchmark tooling): every
|
|
279
|
+
* text block's feature vector and its text, for labeling against ground
|
|
280
|
+
* truth.
|
|
281
|
+
*/
|
|
282
|
+
export function blockTrainingRows(html: string, url: string): { vector: number[], text: string }[] {
|
|
283
|
+
const { blocks, blockVectors } = analyze(html, url)
|
|
284
|
+
return blocks.map((block, index) => ({ vector: blockVectors[index] ?? [], text: block.text }))
|
|
285
|
+
}
|
|
584
286
|
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
// leave nav/footer strings floating in the fallback path. With it, the
|
|
588
|
-
// whole subtree of a forbidden element goes.
|
|
589
|
-
const clean = purify.sanitize(source, { FORBID_TAGS, FORBID_ATTR, KEEP_CONTENT: false }) as string
|
|
287
|
+
/** Render one sanitized candidate to markdown (title prepended when found). */
|
|
288
|
+
export function renderCandidate(candidate: Pick<ScoredCandidate, 'root' | 'title'>): string {
|
|
590
289
|
let markdown: string
|
|
591
290
|
try {
|
|
592
|
-
markdown = turndown.turndown(
|
|
291
|
+
markdown = turndown.turndown(candidate.root)
|
|
593
292
|
} catch {
|
|
594
|
-
markdown =
|
|
293
|
+
markdown = candidate.root.textContent ?? ''
|
|
595
294
|
}
|
|
596
295
|
// Collapse the runs of blank lines nested-list removal leaves behind, then
|
|
597
296
|
// prepend the extracted title when the body did not open with one.
|
|
598
297
|
markdown = markdown.replace(/\n{3,}/g, '\n\n').trim()
|
|
599
|
-
const headingTitle = title?.trim()
|
|
298
|
+
const headingTitle = candidate.title?.trim()
|
|
600
299
|
if (headingTitle !== undefined && headingTitle !== '' && !markdown.startsWith('# ')) {
|
|
601
300
|
markdown = `# ${headingTitle}\n\n${markdown}`
|
|
602
301
|
}
|
|
603
|
-
return
|
|
302
|
+
return markdown
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
/**
|
|
306
|
+
* Convert one rendered HTML document to denoised markdown.
|
|
307
|
+
*
|
|
308
|
+
* Every extraction heuristic proposes a candidate (see `./candidates.ts`);
|
|
309
|
+
* the learned selector (`./selector.ts`) estimates each candidate's quality
|
|
310
|
+
* from page- and candidate-level features and the best one is rendered.
|
|
311
|
+
*
|
|
312
|
+
* @param html - the rendered page HTML (`page.content()`).
|
|
313
|
+
* @param url - the page URL, used to resolve relative links during parsing.
|
|
314
|
+
* @returns the markdown, the extraction mode, and the chosen strategy.
|
|
315
|
+
*/
|
|
316
|
+
export function htmlToMarkdown(html: string, url: string): DenoiseResult {
|
|
317
|
+
const ranked = scoreCandidates(html, url)
|
|
318
|
+
// An empty winner is never better than a non-empty runner-up for a caller
|
|
319
|
+
// that already paid for a browser render.
|
|
320
|
+
const best = ranked.find(candidate => candidate.stats.words > 0) ?? ranked[0]
|
|
321
|
+
if (best === undefined) return { markdown: '', mode: 'document', strategy: 'doc' }
|
|
322
|
+
return {
|
|
323
|
+
markdown: renderCandidate(best),
|
|
324
|
+
mode: DOCUMENT_KINDS.has(best.kind) ? 'document' : 'article',
|
|
325
|
+
strategy: best.kind,
|
|
326
|
+
}
|
|
604
327
|
}
|