dsh-web-fetch-playwright 0.2.19 → 0.2.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -15
- package/README.zh-CN.md +15 -15
- package/cordis.patch.yml +0 -4
- package/lib/client.js +23 -73
- package/lib/index.js +58188 -448
- package/lib/types/autoscroll.d.ts +46 -10
- package/lib/types/block-model.d.ts +2 -0
- package/lib/types/blocks.d.ts +67 -0
- package/lib/types/candidates.d.ts +164 -0
- package/lib/types/client/controller.d.ts +0 -14
- package/lib/types/client/locales.d.ts +1 -1
- package/lib/types/config.d.ts +54 -30
- package/lib/types/index.d.ts +5 -3
- package/lib/types/markdown.d.ts +50 -24
- package/lib/types/provider.d.ts +46 -9
- package/lib/types/selector-model.d.ts +2 -0
- package/lib/types/selector.d.ts +58 -0
- package/lib/types/trees.d.ts +25 -0
- package/lib/types/types.d.ts +7 -6
- package/package.json +1 -1
- package/src/autoscroll.ts +459 -17
- package/src/block-model.ts +5 -0
- package/src/blocks.ts +223 -0
- package/src/candidates.ts +655 -0
- package/src/client/card.tsx +0 -26
- package/src/client/controller.ts +0 -18
- package/src/client/locales.ts +0 -14
- package/src/config.ts +67 -38
- package/src/index.ts +6 -3
- package/src/markdown.ts +136 -413
- package/src/provider.ts +198 -37
- package/src/selector-model.ts +6 -0
- package/src/selector.ts +288 -0
- package/src/trees.ts +43 -0
- package/src/types.ts +7 -7
|
@@ -0,0 +1,655 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Candidate generation for the denoise pipeline.
|
|
3
|
+
*
|
|
4
|
+
* The pipeline used to be ONE chain of hard decisions (Readability → coverage
|
|
5
|
+
* gate → trailing prune → whole-document fallback), each tuned by a global
|
|
6
|
+
* threshold. Every page shape that disagrees with a threshold's notion of
|
|
7
|
+
* "content" (link-dense docs indexes vs. recommendation rails, noscript
|
|
8
|
+
* server fallbacks on forums vs. lazy-image wrappers elsewhere) turned each
|
|
9
|
+
* fix into a regression somewhere else. Here every heuristic instead
|
|
10
|
+
* *proposes* a candidate extraction, and a learned selector
|
|
11
|
+
* (`./selector.ts`) picks one per page from page-level and candidate-level
|
|
12
|
+
* features — adding a heuristic can only add an option, never silently
|
|
13
|
+
* override the others.
|
|
14
|
+
*
|
|
15
|
+
* Candidates (see {@link CandidateKind}):
|
|
16
|
+
* - `art` Readability, trailing link farms pruned (the 0.2.18 article path)
|
|
17
|
+
* - `artu` Readability unpruned (link directories that ARE content)
|
|
18
|
+
* - `artx` Readability with a lenient link-density modifier
|
|
19
|
+
* - `main` the page's semantic main container (`<main>`, `[role=main]`, …)
|
|
20
|
+
* - `doc` the whole sanitized document (the 0.2.18 fallback)
|
|
21
|
+
* - `docc` the whole document minus consent/cookie banners, hidden and
|
|
22
|
+
* chrome-classed subtrees
|
|
23
|
+
* - `rec` the dominant repeated-record region (product cards, list rows)
|
|
24
|
+
* - `ns_art` / `ns_doc` the same two paths after unwrapping `<noscript>`
|
|
25
|
+
* server fallbacks (Discourse and friends ship their whole thread
|
|
26
|
+
* there for script-less clients)
|
|
27
|
+
* - `pre` Discourse `data-preloaded` post stream
|
|
28
|
+
* - `blk` blocks kept by the block classifier (`./blocks.ts`), added by
|
|
29
|
+
* the orchestrator once page features exist
|
|
30
|
+
*
|
|
31
|
+
* @module dsh-web-fetch-playwright/candidates
|
|
32
|
+
*/
|
|
33
|
+
|
|
34
|
+
import { Readability } from '@mozilla/readability'
|
|
35
|
+
|
|
36
|
+
/** Every candidate extraction strategy, in the selector's one-hot order. */
|
|
37
|
+
export const CANDIDATE_KINDS = ['art', 'artu', 'artx', 'main', 'doc', 'docc', 'rec', 'ns_art', 'ns_doc', 'pre', 'blk'] as const
|
|
38
|
+
export type CandidateKind = typeof CANDIDATE_KINDS[number]
|
|
39
|
+
|
|
40
|
+
/** One proposed extraction, still as (unsanitized) HTML. */
|
|
41
|
+
export interface RawCandidate {
|
|
42
|
+
kind: CandidateKind
|
|
43
|
+
/** HTML of the extraction; sanitized by the caller. */
|
|
44
|
+
html: string
|
|
45
|
+
/** Title to prepend when the body does not open with a heading. */
|
|
46
|
+
title?: string
|
|
47
|
+
/** Whether the 0.2.18 pipeline would have returned exactly this candidate. */
|
|
48
|
+
legacy: boolean
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** Layout/noise tags DOMPurify removes outright (with their content). */
|
|
52
|
+
export const FORBID_TAGS = [
|
|
53
|
+
'nav', 'aside', 'header', 'footer', 'form', 'svg', 'iframe', 'noscript',
|
|
54
|
+
'button', 'select', 'option', 'input', 'textarea', 'dialog', 'canvas',
|
|
55
|
+
'video', 'audio', 'template',
|
|
56
|
+
]
|
|
57
|
+
|
|
58
|
+
/** Tags whose subtrees never count as visible text. */
|
|
59
|
+
export const NOISE_SUBTREE_TAGS = new Set([...FORBID_TAGS, 'script', 'style'])
|
|
60
|
+
|
|
61
|
+
// ---------------------------------------------------------------- text ----
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Count non-whitespace characters under `root`, skipping noise subtrees.
|
|
65
|
+
* Whitespace is ignored so CJK and latin pages measure comparable densities.
|
|
66
|
+
*/
|
|
67
|
+
export function visibleTextLength(root: Node | null, skip: Set<string> = NOISE_SUBTREE_TAGS): number {
|
|
68
|
+
if (root === null) return 0
|
|
69
|
+
let length = 0
|
|
70
|
+
const visit = (node: Node): void => {
|
|
71
|
+
if (node.nodeType === 3) {
|
|
72
|
+
length += (node.nodeValue ?? '').replace(/\s+/g, '').length
|
|
73
|
+
return
|
|
74
|
+
}
|
|
75
|
+
if (node.nodeType !== 1 || skip.has(node.nodeName.toLowerCase())) return
|
|
76
|
+
for (let child = node.firstChild; child !== null; child = child.nextSibling) visit(child)
|
|
77
|
+
}
|
|
78
|
+
visit(root)
|
|
79
|
+
return length
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/** Count non-whitespace characters under `root` that sit inside an anchor. */
|
|
83
|
+
export function linkTextLength(root: Node | null): number {
|
|
84
|
+
if (root === null) return 0
|
|
85
|
+
let length = 0
|
|
86
|
+
const visit = (node: Node, insideAnchor: boolean): void => {
|
|
87
|
+
if (node.nodeType === 3) {
|
|
88
|
+
if (insideAnchor) length += (node.nodeValue ?? '').replace(/\s+/g, '').length
|
|
89
|
+
return
|
|
90
|
+
}
|
|
91
|
+
if (node.nodeType !== 1 || NOISE_SUBTREE_TAGS.has(node.nodeName.toLowerCase())) return
|
|
92
|
+
const anchor = insideAnchor || node.nodeName.toLowerCase() === 'a'
|
|
93
|
+
for (let child = node.firstChild; child !== null; child = child.nextSibling) visit(child, anchor)
|
|
94
|
+
}
|
|
95
|
+
visit(root, false)
|
|
96
|
+
return length
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
// ------------------------------------------------------- coverage gate ----
|
|
100
|
+
|
|
101
|
+
/** Below this much visible page text the gate trusts any extraction. */
|
|
102
|
+
const GATE_MIN_BODY_TEXT_CHARS = 1_000
|
|
103
|
+
/** Minimum characters a trusted Readability extraction carries. */
|
|
104
|
+
export const GATE_MIN_ARTICLE_TEXT_CHARS = 250
|
|
105
|
+
/** Minimum share of the page's visible text a trusted extraction covers. */
|
|
106
|
+
const GATE_MIN_ARTICLE_COVERAGE = 0.1
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* The 0.2.18 acceptance gate: distrust an extraction that is tiny in absolute
|
|
110
|
+
* terms or covers a sliver of the page's visible text. Kept as a selector
|
|
111
|
+
* feature and as the legacy-choice marker; it no longer decides alone.
|
|
112
|
+
*/
|
|
113
|
+
export function articleCoversPage(articleHtml: string, document: Document): boolean {
|
|
114
|
+
const bodyText = visibleTextLength(document.body)
|
|
115
|
+
if (bodyText < GATE_MIN_BODY_TEXT_CHARS) return true
|
|
116
|
+
const container = document.createElement('div')
|
|
117
|
+
container.innerHTML = articleHtml
|
|
118
|
+
const articleText = visibleTextLength(container)
|
|
119
|
+
return articleText >= GATE_MIN_ARTICLE_TEXT_CHARS
|
|
120
|
+
&& articleText / bodyText >= GATE_MIN_ARTICLE_COVERAGE
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
// ------------------------------------------------------ trailing prune ----
|
|
124
|
+
|
|
125
|
+
const TRAILING_MIN_TEXT_CHARS = 40
|
|
126
|
+
const TRAILING_LINK_DENSITY = 0.4
|
|
127
|
+
const TRAILING_LINK_COUNT = 3
|
|
128
|
+
const TRAILING_LINK_TEXT_CHARS = 60
|
|
129
|
+
const TRAILING_MAX_ARTICLE_SHARE = 0.6
|
|
130
|
+
const TRAILING_MAX_RUN_SHARE = 0.6
|
|
131
|
+
const TRAILING_UNIFORM_DENSITY_BAND = 0.25
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* Prune trailing link-dense sections (recommendation rails, share bars) from
|
|
135
|
+
* a Readability extraction — the 0.2.18 heuristic (see CHANGELOG 0.2.18 for
|
|
136
|
+
* the 163.com badcase and the Hacker News guards). Since 0.2.20 its output is
|
|
137
|
+
* one candidate (`art`) beside the unpruned extraction (`artu`), so pages
|
|
138
|
+
* whose trailing link block IS content (docs indexes, contributor lists) are
|
|
139
|
+
* no longer forced through it.
|
|
140
|
+
*/
|
|
141
|
+
function pruneTrailingLinkDenseSections(container: Element): void {
|
|
142
|
+
const articleText = visibleTextLength(container)
|
|
143
|
+
if (articleText === 0) return
|
|
144
|
+
for (const element of [container, ...Array.from(container.querySelectorAll('*'))]) {
|
|
145
|
+
if (!container.contains(element) && element !== container) continue
|
|
146
|
+
removeTrailingLinkDenseRun(element, articleText)
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
function removeTrailingLinkDenseRun(parent: Element, articleText: number): void {
|
|
151
|
+
const children = childElements(parent)
|
|
152
|
+
if (children.length === 0) return
|
|
153
|
+
const containerText = visibleTextLength(parent)
|
|
154
|
+
if (containerText === 0) return
|
|
155
|
+
const run: Element[] = []
|
|
156
|
+
let runText = 0
|
|
157
|
+
let runLinked = 0
|
|
158
|
+
let stopIndex = -1
|
|
159
|
+
for (let index = children.length - 1; index >= 0; index--) {
|
|
160
|
+
const child = children[index]
|
|
161
|
+
if (child === undefined) break
|
|
162
|
+
const text = visibleTextLength(child)
|
|
163
|
+
if (text < TRAILING_MIN_TEXT_CHARS) {
|
|
164
|
+
run.push(child)
|
|
165
|
+
continue
|
|
166
|
+
}
|
|
167
|
+
const linked = linkTextLength(child)
|
|
168
|
+
const linkCount = child.querySelectorAll('a').length
|
|
169
|
+
if (
|
|
170
|
+
linked / text >= TRAILING_LINK_DENSITY
|
|
171
|
+
&& (linkCount >= TRAILING_LINK_COUNT || linked >= TRAILING_LINK_TEXT_CHARS)
|
|
172
|
+
&& text <= TRAILING_MAX_ARTICLE_SHARE * articleText
|
|
173
|
+
) {
|
|
174
|
+
run.push(child)
|
|
175
|
+
runText += text
|
|
176
|
+
runLinked += linked
|
|
177
|
+
continue
|
|
178
|
+
}
|
|
179
|
+
stopIndex = index
|
|
180
|
+
break
|
|
181
|
+
}
|
|
182
|
+
if (runText === 0 || stopIndex === -1) return
|
|
183
|
+
const stopChild = children[stopIndex]
|
|
184
|
+
if (stopChild === undefined) return
|
|
185
|
+
const stopDensity = linkTextLength(stopChild) / Math.max(1, visibleTextLength(stopChild))
|
|
186
|
+
if (Math.abs(runLinked / runText - stopDensity) <= TRAILING_UNIFORM_DENSITY_BAND) return
|
|
187
|
+
if (runText > TRAILING_MAX_RUN_SHARE * containerText) return
|
|
188
|
+
for (const child of run) child.remove()
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/** Prune with the 0.2.18 misfire guard; returns the original on a misfire. */
|
|
192
|
+
export function prunedArticleContent(articleHtml: string, document: Document): string {
|
|
193
|
+
const container = document.createElement('div')
|
|
194
|
+
container.innerHTML = articleHtml
|
|
195
|
+
const before = visibleTextLength(container)
|
|
196
|
+
if (before === 0) return articleHtml
|
|
197
|
+
pruneTrailingLinkDenseSections(container)
|
|
198
|
+
const after = visibleTextLength(container)
|
|
199
|
+
if (after === 0 || (before >= GATE_MIN_ARTICLE_TEXT_CHARS && after < GATE_MIN_ARTICLE_TEXT_CHARS)) {
|
|
200
|
+
return articleHtml
|
|
201
|
+
}
|
|
202
|
+
return container.innerHTML
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
// ------------------------------------------------- recommendation rails ----
|
|
206
|
+
|
|
207
|
+
/** Link density at or above which a recommendation-named subtree is a rail. */
|
|
208
|
+
const RAIL_LINK_DENSITY = 0.3
|
|
209
|
+
const RAIL_MIN_LINKS = 4
|
|
210
|
+
/** Prose (low-link text) the page must keep outside the rails. */
|
|
211
|
+
const RAIL_MIN_PROSE_CHARS = 80
|
|
212
|
+
const RAIL_PROSE_LINK_DENSITY = 0.2
|
|
213
|
+
|
|
214
|
+
/**
|
|
215
|
+
* Class/id sub-tokens naming a recommendation / related-content rail: the
|
|
216
|
+
* English words every CMS and theme uses, plus the two syndicated
|
|
217
|
+
* recommendation widgets embedded across thousands of publishers (their
|
|
218
|
+
* containers are named after the vendor, not after what they show). No
|
|
219
|
+
* per-site names: a rail must also pass the link-density and position checks.
|
|
220
|
+
*/
|
|
221
|
+
function isRailName(token: string): boolean {
|
|
222
|
+
return token.includes('recommend') || token.startsWith('related')
|
|
223
|
+
|| token === 'outbrain' || token === 'taboola'
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
function isRecommendationRail(element: Element): boolean {
|
|
227
|
+
const names = `${element.getAttribute('class') ?? ''} ${element.id}`.toLowerCase()
|
|
228
|
+
if (!names.split(/[\s_-]+/).some(isRailName)) return false
|
|
229
|
+
if (element.querySelectorAll('a').length < RAIL_MIN_LINKS) return false
|
|
230
|
+
const text = visibleTextLength(element)
|
|
231
|
+
return text > 0 && linkTextLength(element) / text >= RAIL_LINK_DENSITY
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
/** Marker attribute carried through Readability (it keeps `data-*`). */
|
|
235
|
+
const RAIL_ATTR = 'data-dsh-rail'
|
|
236
|
+
const RAIL_ATTR_TEXT = ` ${RAIL_ATTR}=""`
|
|
237
|
+
|
|
238
|
+
function isProse(block: Element): boolean {
|
|
239
|
+
const text = visibleTextLength(block)
|
|
240
|
+
return text >= RAIL_MIN_PROSE_CHARS && linkTextLength(block) / text < RAIL_PROSE_LINK_DENSITY
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/**
|
|
244
|
+
* A copy of `document` whose recommendation rails ("post_recommends",
|
|
245
|
+
* "related-posts", Outbrain/Taboola widgets) carry a marker attribute, or
|
|
246
|
+
* null when it has none. Readability scores class/id names and text, never
|
|
247
|
+
* `data-*`, so it picks the same container on the copy as on the original —
|
|
248
|
+
* removing the rails beforehand instead was measured to shift its pick to a
|
|
249
|
+
* larger ancestor (WCXB 4206).
|
|
250
|
+
*/
|
|
251
|
+
export function markRecommendationRails(document: Document): Document | null {
|
|
252
|
+
const body = document.body
|
|
253
|
+
if (body === null) return null
|
|
254
|
+
const rails: Element[] = []
|
|
255
|
+
const walk = (element: Element): void => {
|
|
256
|
+
for (const child of childElements(element)) {
|
|
257
|
+
if (isRecommendationRail(child)) rails.push(child)
|
|
258
|
+
else walk(child)
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
walk(body)
|
|
262
|
+
// Most pages have no rail: skip the clone. Otherwise mark the original
|
|
263
|
+
// just long enough to clone it, so the caller's document ends unchanged.
|
|
264
|
+
if (rails.length === 0) return null
|
|
265
|
+
for (const rail of rails) rail.setAttribute(RAIL_ATTR, '')
|
|
266
|
+
const copy = document.cloneNode(true) as Document
|
|
267
|
+
for (const rail of rails) rail.removeAttribute(RAIL_ATTR)
|
|
268
|
+
return copy
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
/**
|
|
272
|
+
* Split a Readability extraction of a rail-marked page into the extraction
|
|
273
|
+
* exactly as it would be unmarked, and — when a marked rail follows prose —
|
|
274
|
+
* the extraction without those rails.
|
|
275
|
+
*
|
|
276
|
+
* The trailing prune cannot catch a rail that outweighs the article (on a
|
|
277
|
+
* short 163.com post the rail is 86% of Readability's pick, past the share
|
|
278
|
+
* guards that keep docs indexes intact), and Readability strips the class
|
|
279
|
+
* names that would tell the two apart, hence the marker. A rail is removed
|
|
280
|
+
* only when it is recognised by name AND link density AND sits after some
|
|
281
|
+
* prose inside the extraction: a category page whose "related-product" grid
|
|
282
|
+
* opens the content (WCXB 4247) is the content, not a rail.
|
|
283
|
+
*/
|
|
284
|
+
export function splitRails(articleHtml: string, document: Document): { plain: string, derailed: string | null } {
|
|
285
|
+
if (!articleHtml.includes(RAIL_ATTR)) return { plain: articleHtml, derailed: null }
|
|
286
|
+
const plain = articleHtml.split(RAIL_ATTR_TEXT).join('')
|
|
287
|
+
const container = htmlFragment(articleHtml, document)
|
|
288
|
+
const prose = Array.from(container.querySelectorAll('p, div, td, section, article')).filter(block => !block.hasAttribute(RAIL_ATTR) && block.closest(`[${RAIL_ATTR}]`) === null && isProse(block))
|
|
289
|
+
let removed = 0
|
|
290
|
+
for (const rail of Array.from(container.querySelectorAll(`[${RAIL_ATTR}]`))) {
|
|
291
|
+
// eslint-disable-next-line no-bitwise
|
|
292
|
+
const preceded = prose.some(block => (block.compareDocumentPosition(rail) & 4 /* FOLLOWING */) !== 0 && !block.contains(rail))
|
|
293
|
+
if (preceded) {
|
|
294
|
+
rail.remove()
|
|
295
|
+
removed++
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
if (removed === 0 || visibleTextLength(container) < RAIL_MIN_PROSE_CHARS) return { plain, derailed: null }
|
|
299
|
+
return { plain, derailed: container.innerHTML.split(RAIL_ATTR_TEXT).join('') }
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
// ------------------------------------------------------------ helpers ----
|
|
303
|
+
|
|
304
|
+
/**
|
|
305
|
+
* An element's element children as an array. Walks sibling pointers:
|
|
306
|
+
* jsdom's `HTMLCollection` is a proxy whose indexed access dominates
|
|
307
|
+
* profiles when `Array.from(el.children)` runs on every element of a large
|
|
308
|
+
* page (measured: 5.2s for a 4,000-paragraph article).
|
|
309
|
+
*/
|
|
310
|
+
export function childElements(element: Element): Element[] {
|
|
311
|
+
const out: Element[] = []
|
|
312
|
+
for (let child = element.firstElementChild; child !== null; child = child.nextElementSibling) out.push(child)
|
|
313
|
+
return out
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
function htmlFragment(html: string, document: Document): Element {
|
|
317
|
+
const container = document.createElement('div')
|
|
318
|
+
container.innerHTML = html
|
|
319
|
+
return container
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
interface ReadabilityPick { content: string, title?: string }
|
|
323
|
+
|
|
324
|
+
/** Run Readability on a clone of `document`; null on failure/empty. */
|
|
325
|
+
function runReadability(document: Document, options?: ConstructorParameters<typeof Readability>[1]): ReadabilityPick | null {
|
|
326
|
+
try {
|
|
327
|
+
const cloned = document.cloneNode(true) as Document
|
|
328
|
+
const article = new Readability(cloned, options).parse()
|
|
329
|
+
if (article === null || typeof article.content !== 'string' || article.content === '') return null
|
|
330
|
+
return { content: article.content, title: article.title ?? undefined }
|
|
331
|
+
} catch {
|
|
332
|
+
return null
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
/**
|
|
337
|
+
* Rewrite every `<noscript>` into a `<div>` holding its markup.
|
|
338
|
+
*
|
|
339
|
+
* Script-less parsing (jsdom, static snapshots) exposes noscript content as
|
|
340
|
+
* markup that DOMPurify and Readability then delete wholesale — on Discourse
|
|
341
|
+
* that is the entire server-rendered thread, measured 16–36KB per page.
|
|
342
|
+
* Scripting-enabled parsing leaves it as one raw text node, which is
|
|
343
|
+
* reparsed here so both parser modes converge.
|
|
344
|
+
*/
|
|
345
|
+
export function unwrapNoscripts(document: Document): number {
|
|
346
|
+
let count = 0
|
|
347
|
+
for (const noscript of Array.from(document.querySelectorAll('noscript'))) {
|
|
348
|
+
const div = document.createElement('div')
|
|
349
|
+
const onlyText = noscript.children.length === 0
|
|
350
|
+
if (onlyText) div.innerHTML = noscript.textContent ?? ''
|
|
351
|
+
else while (noscript.firstChild !== null) div.appendChild(noscript.firstChild)
|
|
352
|
+
noscript.replaceWith(div)
|
|
353
|
+
count++
|
|
354
|
+
}
|
|
355
|
+
return count
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
/** Visible text inside `<noscript>` elements (parsed either way). */
|
|
359
|
+
export function noscriptTextLength(document: Document): number {
|
|
360
|
+
let total = 0
|
|
361
|
+
for (const noscript of Array.from(document.querySelectorAll('noscript'))) {
|
|
362
|
+
if (noscript.children.length === 0) {
|
|
363
|
+
const div = document.createElement('div')
|
|
364
|
+
div.innerHTML = noscript.textContent ?? ''
|
|
365
|
+
total += visibleTextLength(div)
|
|
366
|
+
} else {
|
|
367
|
+
total += visibleTextLength(noscript, new Set(['script', 'style']))
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
return total
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
/** Semantic main-container selectors, most specific first. */
|
|
374
|
+
const MAIN_SELECTORS = [
|
|
375
|
+
'main', '[role="main"]', '#main-content', '#maincontent', '#main', '.main-content',
|
|
376
|
+
'#content', '#primary', '.content',
|
|
377
|
+
]
|
|
378
|
+
|
|
379
|
+
/** Find the page's semantic main container, if it carries real text. */
|
|
380
|
+
function findMainContainer(document: Document): Element | null {
|
|
381
|
+
const articles = document.querySelectorAll('article')
|
|
382
|
+
for (const selector of MAIN_SELECTORS) {
|
|
383
|
+
const element = document.querySelector(selector)
|
|
384
|
+
if (element !== null && visibleTextLength(element) >= 200) return element
|
|
385
|
+
}
|
|
386
|
+
if (articles.length === 1 && articles[0] !== undefined && visibleTextLength(articles[0]) >= 200) return articles[0]
|
|
387
|
+
return null
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
/** Sub-tokens of class/id names that mark page chrome, not content. */
|
|
391
|
+
const CHROME_TOKENS = new Set([
|
|
392
|
+
'cookie', 'cookies', 'cookiebar', 'consent', 'gdpr', 'onetrust', 'cmp', 'ccpa',
|
|
393
|
+
'newsletter', 'subscribe', 'popup', 'modal', 'overlay', 'lightbox', 'skip', 'skiplink',
|
|
394
|
+
'breadcrumb', 'breadcrumbs', 'share', 'sharing', 'social', 'socials', 'advert',
|
|
395
|
+
'advertisement', 'ads', 'ad', 'sponsor', 'sponsored', 'navbar', 'navigation', 'megamenu',
|
|
396
|
+
'menu', 'sidebar', 'promo', 'banner', 'toast', 'drawer', 'offcanvas',
|
|
397
|
+
])
|
|
398
|
+
|
|
399
|
+
const CHROME_ROLES = new Set(['navigation', 'dialog', 'alertdialog', 'complementary', 'contentinfo', 'search', 'banner'])
|
|
400
|
+
|
|
401
|
+
/**
|
|
402
|
+
* Cookie/consent banner wording (EN/DE/FR/ES/IT/NL/ZH), as a regex source
|
|
403
|
+
* shared with the selector's `consent_hits` feature so both read the same
|
|
404
|
+
* languages. Chinese banners say "Cookie" or 隐私; 同意 ("agree") alone is
|
|
405
|
+
* everyday prose and only counts as the banner's ACTION below.
|
|
406
|
+
*/
|
|
407
|
+
export const CONSENT_TERMS = String.raw`\b(cookies?|consent|datenschutz\w*|einwilligung|zustimm\w*|akzeptier\w*|privacy|gdpr|rgpd)\b|隐私`
|
|
408
|
+
const CONSENT_TEXT = new RegExp(CONSENT_TERMS, 'i')
|
|
409
|
+
const CONSENT_ACTION = /\b(accept|agree|allow|reject|decline|manage|preferences|settings|akzeptieren|zustimmen|einverstanden|ablehnen|accepter|aceptar|accetta|accepteren)\b|接受|同意/i
|
|
410
|
+
|
|
411
|
+
function isChromeElement(element: Element): boolean {
|
|
412
|
+
const role = element.getAttribute('role')
|
|
413
|
+
if (role !== null && CHROME_ROLES.has(role.toLowerCase())) return true
|
|
414
|
+
if (element.hasAttribute('hidden') || element.getAttribute('aria-hidden') === 'true' || element.getAttribute('aria-modal') === 'true') return true
|
|
415
|
+
const style = element.getAttribute('style')
|
|
416
|
+
if (style !== null && /(display\s*:\s*none|visibility\s*:\s*hidden)/i.test(style)) return true
|
|
417
|
+
const names = `${element.getAttribute('class') ?? ''} ${element.id}`.toLowerCase()
|
|
418
|
+
if (names.trim() === '') return false
|
|
419
|
+
for (const token of names.split(/[\s_-]+/)) {
|
|
420
|
+
if (CHROME_TOKENS.has(token)) return true
|
|
421
|
+
}
|
|
422
|
+
return false
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
/**
|
|
426
|
+
* Strip page chrome from a cloned body: chrome-named/roled subtrees, hidden
|
|
427
|
+
* subtrees, and consent banners recognised by their wording. A subtree is
|
|
428
|
+
* only removed while it is a minority of the page, so a site that names its
|
|
429
|
+
* article wrapper `content-modal` cannot lose everything.
|
|
430
|
+
*/
|
|
431
|
+
export function stripPageChrome(body: Element): void {
|
|
432
|
+
const total = Math.max(1, visibleTextLength(body))
|
|
433
|
+
const doomed: Element[] = []
|
|
434
|
+
const walk = (element: Element): void => {
|
|
435
|
+
for (const child of childElements(element)) {
|
|
436
|
+
if (isChromeElement(child) && visibleTextLength(child) <= 0.4 * total) {
|
|
437
|
+
doomed.push(child)
|
|
438
|
+
continue
|
|
439
|
+
}
|
|
440
|
+
walk(child)
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
walk(body)
|
|
444
|
+
for (const element of doomed) element.remove()
|
|
445
|
+
// Consent banners without telltale names: climb from the wording to the
|
|
446
|
+
// largest ancestor that is still banner-sized.
|
|
447
|
+
const ownerDocument = body.ownerDocument
|
|
448
|
+
const walker = ownerDocument.createTreeWalker(body, 4 /* SHOW_TEXT */)
|
|
449
|
+
const hits: Element[] = []
|
|
450
|
+
for (let node = walker.nextNode(); node !== null; node = walker.nextNode()) {
|
|
451
|
+
const text = node.nodeValue ?? ''
|
|
452
|
+
if (text.length < 20 || !CONSENT_TEXT.test(text)) continue
|
|
453
|
+
let block = node.parentElement
|
|
454
|
+
if (block === null) continue
|
|
455
|
+
const blockText = block.textContent ?? ''
|
|
456
|
+
if (!CONSENT_ACTION.test(blockText) && !CONSENT_ACTION.test(block.parentElement?.textContent ?? '')) continue
|
|
457
|
+
while (block.parentElement !== null && block.parentElement !== body && visibleTextLength(block.parentElement) < 1_200) {
|
|
458
|
+
block = block.parentElement
|
|
459
|
+
}
|
|
460
|
+
if (visibleTextLength(block) < 1_200) hits.push(block)
|
|
461
|
+
}
|
|
462
|
+
for (const element of hits) if (element.isConnected) element.remove()
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
/** Record-region discovery result. */
|
|
466
|
+
export interface RecordRegion {
|
|
467
|
+
items: Element[]
|
|
468
|
+
/** Total visible text of the items. */
|
|
469
|
+
text: number
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
/**
|
|
473
|
+
* Find the dominant repeated-record region: the sibling group with the same
|
|
474
|
+
* tag + class signature, at least four members, carrying the most text —
|
|
475
|
+
* product cards on a collection page, rows on a listing. Menu-like groups
|
|
476
|
+
* (tiny items, flat subtrees) are skipped.
|
|
477
|
+
*/
|
|
478
|
+
export function findRecordRegion(body: Element): RecordRegion | null {
|
|
479
|
+
let best: RecordRegion | null = null
|
|
480
|
+
const visit = (element: Element): void => {
|
|
481
|
+
const name = element.nodeName.toLowerCase()
|
|
482
|
+
if (NOISE_SUBTREE_TAGS.has(name)) return
|
|
483
|
+
const children = childElements(element)
|
|
484
|
+
if (children.length >= 4) {
|
|
485
|
+
const groups = new Map<string, Element[]>()
|
|
486
|
+
for (const child of children) {
|
|
487
|
+
const classes = (child.getAttribute('class') ?? '').replace(/\d+/g, '').trim().split(/\s+/).filter(Boolean).sort().slice(0, 2).join('.')
|
|
488
|
+
const signature = `${child.nodeName}.${classes}`
|
|
489
|
+
const group = groups.get(signature)
|
|
490
|
+
if (group === undefined) groups.set(signature, [child])
|
|
491
|
+
else group.push(child)
|
|
492
|
+
}
|
|
493
|
+
for (const items of groups.values()) {
|
|
494
|
+
if (items.length < 4) continue
|
|
495
|
+
let text = 0
|
|
496
|
+
let depth = 0
|
|
497
|
+
for (const item of items) {
|
|
498
|
+
text += visibleTextLength(item)
|
|
499
|
+
depth += item.getElementsByTagName('*').length
|
|
500
|
+
}
|
|
501
|
+
if (text / items.length < 15 || depth / items.length < 2) continue
|
|
502
|
+
if (best === null || text > best.text) best = { items, text }
|
|
503
|
+
}
|
|
504
|
+
}
|
|
505
|
+
for (const child of children) visit(child)
|
|
506
|
+
}
|
|
507
|
+
visit(body)
|
|
508
|
+
return best
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
/**
|
|
512
|
+
* A post's timestamp as a locale-neutral calendar date (`2024-09-05`, UTC).
|
|
513
|
+
* Deliberately not any UI's localized rendering: the reader may be any
|
|
514
|
+
* language, and the page itself prints dates in its own locale.
|
|
515
|
+
*/
|
|
516
|
+
function formatForumDate(iso: unknown): string {
|
|
517
|
+
if (typeof iso !== 'string') return ''
|
|
518
|
+
const date = new Date(iso)
|
|
519
|
+
if (Number.isNaN(date.getTime())) return ''
|
|
520
|
+
return date.toISOString().slice(0, 10)
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
/**
|
|
524
|
+
* Discourse ships the topic's post stream as JSON in
|
|
525
|
+
* `#data-preloaded[data-preloaded]` (a map of JSON strings). Rebuild the
|
|
526
|
+
* thread as HTML: author, date, cooked post body.
|
|
527
|
+
*/
|
|
528
|
+
export function discoursePreloadedHtml(document: Document): string | null {
|
|
529
|
+
const holder = document.querySelector('#data-preloaded[data-preloaded]')
|
|
530
|
+
const raw = holder?.getAttribute('data-preloaded')
|
|
531
|
+
if (raw == null) return null
|
|
532
|
+
try {
|
|
533
|
+
const outer = JSON.parse(raw) as Record<string, unknown>
|
|
534
|
+
for (const [key, value] of Object.entries(outer)) {
|
|
535
|
+
if (!key.startsWith('topic_') || typeof value !== 'string') continue
|
|
536
|
+
const topic = JSON.parse(value) as { title?: string, post_stream?: { posts?: { username?: string, created_at?: string, cooked?: string }[] } }
|
|
537
|
+
const posts = topic.post_stream?.posts ?? []
|
|
538
|
+
if (posts.length === 0) continue
|
|
539
|
+
const parts = posts.map(post => `<div><p>${escapeHtml(post.username ?? '')}</p><p>${escapeHtml(formatForumDate(post.created_at))}</p>${post.cooked ?? ''}</div>`)
|
|
540
|
+
return parts.join('\n')
|
|
541
|
+
}
|
|
542
|
+
} catch {
|
|
543
|
+
return null
|
|
544
|
+
}
|
|
545
|
+
return null
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
function escapeHtml(text: string): string {
|
|
549
|
+
return text.replace(/[&<>"]/g, ch => ({ '&': '&', '<': '<', '>': '>', '"': '"' })[ch] ?? ch)
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
// ---------------------------------------------------------- generation ----
|
|
553
|
+
|
|
554
|
+
/** Everything candidate generation learned about the page, for features. */
|
|
555
|
+
export interface GenerationFacts {
|
|
556
|
+
readabilityNull: boolean
|
|
557
|
+
gatePass: boolean
|
|
558
|
+
pruneChanged: boolean
|
|
559
|
+
noscriptText: number
|
|
560
|
+
record: RecordRegion | null
|
|
561
|
+
hasMain: boolean
|
|
562
|
+
hasPreloaded: boolean
|
|
563
|
+
/** The semantic main container, when found. */
|
|
564
|
+
main: Element | null
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
/** Below this much noscript text the noscript-unwrapped paths are skipped. */
|
|
568
|
+
const NOSCRIPT_MIN_TEXT_CHARS = 200
|
|
569
|
+
|
|
570
|
+
/**
|
|
571
|
+
* Propose every candidate extraction for one parsed page.
|
|
572
|
+
* @param document - the parsed page (data URIs already elided); not mutated.
|
|
573
|
+
*/
|
|
574
|
+
export function generateCandidates(document: Document): { candidates: RawCandidate[], facts: GenerationFacts } {
|
|
575
|
+
const candidates: RawCandidate[] = []
|
|
576
|
+
const body = document.body
|
|
577
|
+
const bodyHtml = body?.innerHTML ?? ''
|
|
578
|
+
|
|
579
|
+
// Rails are marked before Readability (it drops the class names that
|
|
580
|
+
// identify them) and cut from its output; the gate and `artu` see the
|
|
581
|
+
// extraction exactly as unmarked, so the legacy marker and the "keep
|
|
582
|
+
// everything" alternative are unchanged.
|
|
583
|
+
const marked = runReadability(markRecommendationRails(document) ?? document)
|
|
584
|
+
const rails = marked === null ? null : splitRails(marked.content, document)
|
|
585
|
+
const art = marked === null || rails === null ? null : { ...marked, content: rails.plain }
|
|
586
|
+
const gatePass = art !== null && articleCoversPage(art.content, document)
|
|
587
|
+
let pruneChanged = false
|
|
588
|
+
if (art !== null) {
|
|
589
|
+
const pruned = prunedArticleContent(rails?.derailed ?? art.content, document)
|
|
590
|
+
pruneChanged = pruned !== art.content
|
|
591
|
+
candidates.push({ kind: 'art', html: pruned, title: art.title, legacy: gatePass })
|
|
592
|
+
if (pruneChanged) candidates.push({ kind: 'artu', html: art.content, title: art.title, legacy: false })
|
|
593
|
+
}
|
|
594
|
+
// `linkDensityModifier` is a public Readability option missing from the
|
|
595
|
+
// bundled type declarations; it relaxes the link-density cleaning cutoffs.
|
|
596
|
+
const lenient = runReadability(document, { linkDensityModifier: 0.4 } as ConstructorParameters<typeof Readability>[1])
|
|
597
|
+
if (lenient !== null && lenient.content !== art?.content) {
|
|
598
|
+
candidates.push({ kind: 'artx', html: lenient.content, title: lenient.title, legacy: false })
|
|
599
|
+
}
|
|
600
|
+
const title = art?.title ?? lenient?.title
|
|
601
|
+
|
|
602
|
+
const main = body === null ? null : findMainContainer(document)
|
|
603
|
+
if (main !== null) candidates.push({ kind: 'main', html: main.innerHTML, title, legacy: false })
|
|
604
|
+
|
|
605
|
+
candidates.push({ kind: 'doc', html: bodyHtml, legacy: !gatePass })
|
|
606
|
+
|
|
607
|
+
if (body !== null) {
|
|
608
|
+
const cleaned = body.cloneNode(true) as Element
|
|
609
|
+
stripPageChrome(cleaned)
|
|
610
|
+
candidates.push({ kind: 'docc', html: cleaned.innerHTML, legacy: false })
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
const record = body === null ? null : findRecordRegion(body)
|
|
614
|
+
if (record !== null) {
|
|
615
|
+
const heading = document.querySelector('h1')
|
|
616
|
+
const html = (heading !== null ? heading.outerHTML : '') + record.items.map(item => item.outerHTML).join('\n')
|
|
617
|
+
candidates.push({ kind: 'rec', html, legacy: false })
|
|
618
|
+
}
|
|
619
|
+
|
|
620
|
+
const noscriptText = noscriptTextLength(document)
|
|
621
|
+
if (noscriptText >= NOSCRIPT_MIN_TEXT_CHARS) {
|
|
622
|
+
const unwrapped = document.cloneNode(true) as Document
|
|
623
|
+
unwrapNoscripts(unwrapped)
|
|
624
|
+
const nsArt = runReadability(unwrapped)
|
|
625
|
+
if (nsArt !== null) candidates.push({ kind: 'ns_art', html: prunedArticleContent(nsArt.content, unwrapped), title: nsArt.title, legacy: false })
|
|
626
|
+
candidates.push({ kind: 'ns_doc', html: unwrapped.body?.innerHTML ?? '', legacy: false })
|
|
627
|
+
}
|
|
628
|
+
|
|
629
|
+
const preloaded = discoursePreloadedHtml(document)
|
|
630
|
+
if (preloaded !== null) candidates.push({ kind: 'pre', html: preloaded, title, legacy: false })
|
|
631
|
+
|
|
632
|
+
// Identical proposals (a `<main>` that IS Readability's pick, a chrome
|
|
633
|
+
// strip that removed nothing) would only cost another sanitize pass.
|
|
634
|
+
const seen = new Set(candidates.filter(candidate => candidate.legacy).map(candidate => candidate.html))
|
|
635
|
+
const unique = candidates.filter(candidate => {
|
|
636
|
+
if (candidate.legacy) return true
|
|
637
|
+
if (seen.has(candidate.html)) return false
|
|
638
|
+
seen.add(candidate.html)
|
|
639
|
+
return true
|
|
640
|
+
})
|
|
641
|
+
|
|
642
|
+
return {
|
|
643
|
+
candidates: unique,
|
|
644
|
+
facts: {
|
|
645
|
+
readabilityNull: art === null,
|
|
646
|
+
gatePass,
|
|
647
|
+
pruneChanged,
|
|
648
|
+
noscriptText,
|
|
649
|
+
record,
|
|
650
|
+
hasMain: main !== null,
|
|
651
|
+
hasPreloaded: preloaded !== null,
|
|
652
|
+
main,
|
|
653
|
+
},
|
|
654
|
+
}
|
|
655
|
+
}
|
package/src/client/card.tsx
CHANGED
|
@@ -177,32 +177,6 @@ export function PlaywrightCard(props: PlaywrightCardProps) {
|
|
|
177
177
|
onEdit={(text) => { props.edit('maxConcurrency', text) }}
|
|
178
178
|
onReset={() => { props.resetField('maxConcurrency') }}
|
|
179
179
|
/>
|
|
180
|
-
<ValueField
|
|
181
|
-
id="plugin-config-playwright-challenge-wait"
|
|
182
|
-
label={t('challengeWaitMs')}
|
|
183
|
-
hint={t('challengeWaitMsHint')}
|
|
184
|
-
placeholder={t('challengeWaitMsPlaceholder')}
|
|
185
|
-
overriddenLabel={t('overridden')}
|
|
186
|
-
resetLabel={t('reset')}
|
|
187
|
-
invalidLabel={t('invalidText')}
|
|
188
|
-
disabled={disabled}
|
|
189
|
-
{...state.challengeWaitMs}
|
|
190
|
-
onEdit={(text) => { props.edit('challengeWaitMs', text) }}
|
|
191
|
-
onReset={() => { props.resetField('challengeWaitMs') }}
|
|
192
|
-
/>
|
|
193
|
-
<ValueField
|
|
194
|
-
id="plugin-config-playwright-scroll-budget"
|
|
195
|
-
label={t('scrollBudgetMs')}
|
|
196
|
-
hint={t('scrollBudgetMsHint')}
|
|
197
|
-
placeholder={t('scrollBudgetMsPlaceholder')}
|
|
198
|
-
overriddenLabel={t('overridden')}
|
|
199
|
-
resetLabel={t('reset')}
|
|
200
|
-
invalidLabel={t('invalidText')}
|
|
201
|
-
disabled={disabled}
|
|
202
|
-
{...state.scrollBudgetMs}
|
|
203
|
-
onEdit={(text) => { props.edit('scrollBudgetMs', text) }}
|
|
204
|
-
onReset={() => { props.resetField('scrollBudgetMs') }}
|
|
205
|
-
/>
|
|
206
180
|
</CardForm>
|
|
207
181
|
)
|
|
208
182
|
}
|