dsh-web-fetch-playwright 0.2.19 → 0.2.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,655 @@
1
+ /**
2
+ * Candidate generation for the denoise pipeline.
3
+ *
4
+ * The pipeline used to be ONE chain of hard decisions (Readability → coverage
5
+ * gate → trailing prune → whole-document fallback), each tuned by a global
6
+ * threshold. Every page shape that disagrees with a threshold's notion of
7
+ * "content" (link-dense docs indexes vs. recommendation rails, noscript
8
+ * server fallbacks on forums vs. lazy-image wrappers elsewhere) turned each
9
+ * fix into a regression somewhere else. Here every heuristic instead
10
+ * *proposes* a candidate extraction, and a learned selector
11
+ * (`./selector.ts`) picks one per page from page-level and candidate-level
12
+ * features — adding a heuristic can only add an option, never silently
13
+ * override the others.
14
+ *
15
+ * Candidates (see {@link CandidateKind}):
16
+ * - `art` Readability, trailing link farms pruned (the 0.2.18 article path)
17
+ * - `artu` Readability unpruned (link directories that ARE content)
18
+ * - `artx` Readability with a lenient link-density modifier
19
+ * - `main` the page's semantic main container (`<main>`, `[role=main]`, …)
20
+ * - `doc` the whole sanitized document (the 0.2.18 fallback)
21
+ * - `docc` the whole document minus consent/cookie banners, hidden and
22
+ * chrome-classed subtrees
23
+ * - `rec` the dominant repeated-record region (product cards, list rows)
24
+ * - `ns_art` / `ns_doc` the same two paths after unwrapping `<noscript>`
25
+ * server fallbacks (Discourse and friends ship their whole thread
26
+ * there for script-less clients)
27
+ * - `pre` Discourse `data-preloaded` post stream
28
+ * - `blk` blocks kept by the block classifier (`./blocks.ts`), added by
29
+ * the orchestrator once page features exist
30
+ *
31
+ * @module dsh-web-fetch-playwright/candidates
32
+ */
33
+
34
+ import { Readability } from '@mozilla/readability'
35
+
36
+ /** Every candidate extraction strategy, in the selector's one-hot order. */
37
+ export const CANDIDATE_KINDS = ['art', 'artu', 'artx', 'main', 'doc', 'docc', 'rec', 'ns_art', 'ns_doc', 'pre', 'blk'] as const
38
+ export type CandidateKind = typeof CANDIDATE_KINDS[number]
39
+
40
+ /** One proposed extraction, still as (unsanitized) HTML. */
41
+ export interface RawCandidate {
42
+ kind: CandidateKind
43
+ /** HTML of the extraction; sanitized by the caller. */
44
+ html: string
45
+ /** Title to prepend when the body does not open with a heading. */
46
+ title?: string
47
+ /** Whether the 0.2.18 pipeline would have returned exactly this candidate. */
48
+ legacy: boolean
49
+ }
50
+
51
+ /** Layout/noise tags DOMPurify removes outright (with their content). */
52
+ export const FORBID_TAGS = [
53
+ 'nav', 'aside', 'header', 'footer', 'form', 'svg', 'iframe', 'noscript',
54
+ 'button', 'select', 'option', 'input', 'textarea', 'dialog', 'canvas',
55
+ 'video', 'audio', 'template',
56
+ ]
57
+
58
+ /** Tags whose subtrees never count as visible text. */
59
+ export const NOISE_SUBTREE_TAGS = new Set([...FORBID_TAGS, 'script', 'style'])
60
+
61
+ // ---------------------------------------------------------------- text ----
62
+
63
+ /**
64
+ * Count non-whitespace characters under `root`, skipping noise subtrees.
65
+ * Whitespace is ignored so CJK and latin pages measure comparable densities.
66
+ */
67
+ export function visibleTextLength(root: Node | null, skip: Set<string> = NOISE_SUBTREE_TAGS): number {
68
+ if (root === null) return 0
69
+ let length = 0
70
+ const visit = (node: Node): void => {
71
+ if (node.nodeType === 3) {
72
+ length += (node.nodeValue ?? '').replace(/\s+/g, '').length
73
+ return
74
+ }
75
+ if (node.nodeType !== 1 || skip.has(node.nodeName.toLowerCase())) return
76
+ for (let child = node.firstChild; child !== null; child = child.nextSibling) visit(child)
77
+ }
78
+ visit(root)
79
+ return length
80
+ }
81
+
82
+ /** Count non-whitespace characters under `root` that sit inside an anchor. */
83
+ export function linkTextLength(root: Node | null): number {
84
+ if (root === null) return 0
85
+ let length = 0
86
+ const visit = (node: Node, insideAnchor: boolean): void => {
87
+ if (node.nodeType === 3) {
88
+ if (insideAnchor) length += (node.nodeValue ?? '').replace(/\s+/g, '').length
89
+ return
90
+ }
91
+ if (node.nodeType !== 1 || NOISE_SUBTREE_TAGS.has(node.nodeName.toLowerCase())) return
92
+ const anchor = insideAnchor || node.nodeName.toLowerCase() === 'a'
93
+ for (let child = node.firstChild; child !== null; child = child.nextSibling) visit(child, anchor)
94
+ }
95
+ visit(root, false)
96
+ return length
97
+ }
98
+
99
+ // ------------------------------------------------------- coverage gate ----
100
+
101
+ /** Below this much visible page text the gate trusts any extraction. */
102
+ const GATE_MIN_BODY_TEXT_CHARS = 1_000
103
+ /** Minimum characters a trusted Readability extraction carries. */
104
+ export const GATE_MIN_ARTICLE_TEXT_CHARS = 250
105
+ /** Minimum share of the page's visible text a trusted extraction covers. */
106
+ const GATE_MIN_ARTICLE_COVERAGE = 0.1
107
+
108
+ /**
109
+ * The 0.2.18 acceptance gate: distrust an extraction that is tiny in absolute
110
+ * terms or covers a sliver of the page's visible text. Kept as a selector
111
+ * feature and as the legacy-choice marker; it no longer decides alone.
112
+ */
113
+ export function articleCoversPage(articleHtml: string, document: Document): boolean {
114
+ const bodyText = visibleTextLength(document.body)
115
+ if (bodyText < GATE_MIN_BODY_TEXT_CHARS) return true
116
+ const container = document.createElement('div')
117
+ container.innerHTML = articleHtml
118
+ const articleText = visibleTextLength(container)
119
+ return articleText >= GATE_MIN_ARTICLE_TEXT_CHARS
120
+ && articleText / bodyText >= GATE_MIN_ARTICLE_COVERAGE
121
+ }
122
+
123
+ // ------------------------------------------------------ trailing prune ----
124
+
125
+ const TRAILING_MIN_TEXT_CHARS = 40
126
+ const TRAILING_LINK_DENSITY = 0.4
127
+ const TRAILING_LINK_COUNT = 3
128
+ const TRAILING_LINK_TEXT_CHARS = 60
129
+ const TRAILING_MAX_ARTICLE_SHARE = 0.6
130
+ const TRAILING_MAX_RUN_SHARE = 0.6
131
+ const TRAILING_UNIFORM_DENSITY_BAND = 0.25
132
+
133
+ /**
134
+ * Prune trailing link-dense sections (recommendation rails, share bars) from
135
+ * a Readability extraction — the 0.2.18 heuristic (see CHANGELOG 0.2.18 for
136
+ * the 163.com badcase and the Hacker News guards). Since 0.2.20 its output is
137
+ * one candidate (`art`) beside the unpruned extraction (`artu`), so pages
138
+ * whose trailing link block IS content (docs indexes, contributor lists) are
139
+ * no longer forced through it.
140
+ */
141
+ function pruneTrailingLinkDenseSections(container: Element): void {
142
+ const articleText = visibleTextLength(container)
143
+ if (articleText === 0) return
144
+ for (const element of [container, ...Array.from(container.querySelectorAll('*'))]) {
145
+ if (!container.contains(element) && element !== container) continue
146
+ removeTrailingLinkDenseRun(element, articleText)
147
+ }
148
+ }
149
+
150
+ function removeTrailingLinkDenseRun(parent: Element, articleText: number): void {
151
+ const children = childElements(parent)
152
+ if (children.length === 0) return
153
+ const containerText = visibleTextLength(parent)
154
+ if (containerText === 0) return
155
+ const run: Element[] = []
156
+ let runText = 0
157
+ let runLinked = 0
158
+ let stopIndex = -1
159
+ for (let index = children.length - 1; index >= 0; index--) {
160
+ const child = children[index]
161
+ if (child === undefined) break
162
+ const text = visibleTextLength(child)
163
+ if (text < TRAILING_MIN_TEXT_CHARS) {
164
+ run.push(child)
165
+ continue
166
+ }
167
+ const linked = linkTextLength(child)
168
+ const linkCount = child.querySelectorAll('a').length
169
+ if (
170
+ linked / text >= TRAILING_LINK_DENSITY
171
+ && (linkCount >= TRAILING_LINK_COUNT || linked >= TRAILING_LINK_TEXT_CHARS)
172
+ && text <= TRAILING_MAX_ARTICLE_SHARE * articleText
173
+ ) {
174
+ run.push(child)
175
+ runText += text
176
+ runLinked += linked
177
+ continue
178
+ }
179
+ stopIndex = index
180
+ break
181
+ }
182
+ if (runText === 0 || stopIndex === -1) return
183
+ const stopChild = children[stopIndex]
184
+ if (stopChild === undefined) return
185
+ const stopDensity = linkTextLength(stopChild) / Math.max(1, visibleTextLength(stopChild))
186
+ if (Math.abs(runLinked / runText - stopDensity) <= TRAILING_UNIFORM_DENSITY_BAND) return
187
+ if (runText > TRAILING_MAX_RUN_SHARE * containerText) return
188
+ for (const child of run) child.remove()
189
+ }
190
+
191
+ /** Prune with the 0.2.18 misfire guard; returns the original on a misfire. */
192
+ export function prunedArticleContent(articleHtml: string, document: Document): string {
193
+ const container = document.createElement('div')
194
+ container.innerHTML = articleHtml
195
+ const before = visibleTextLength(container)
196
+ if (before === 0) return articleHtml
197
+ pruneTrailingLinkDenseSections(container)
198
+ const after = visibleTextLength(container)
199
+ if (after === 0 || (before >= GATE_MIN_ARTICLE_TEXT_CHARS && after < GATE_MIN_ARTICLE_TEXT_CHARS)) {
200
+ return articleHtml
201
+ }
202
+ return container.innerHTML
203
+ }
204
+
205
+ // ------------------------------------------------- recommendation rails ----
206
+
207
+ /** Link density at or above which a recommendation-named subtree is a rail. */
208
+ const RAIL_LINK_DENSITY = 0.3
209
+ const RAIL_MIN_LINKS = 4
210
+ /** Prose (low-link text) the page must keep outside the rails. */
211
+ const RAIL_MIN_PROSE_CHARS = 80
212
+ const RAIL_PROSE_LINK_DENSITY = 0.2
213
+
214
+ /**
215
+ * Class/id sub-tokens naming a recommendation / related-content rail: the
216
+ * English words every CMS and theme uses, plus the two syndicated
217
+ * recommendation widgets embedded across thousands of publishers (their
218
+ * containers are named after the vendor, not after what they show). No
219
+ * per-site names: a rail must also pass the link-density and position checks.
220
+ */
221
+ function isRailName(token: string): boolean {
222
+ return token.includes('recommend') || token.startsWith('related')
223
+ || token === 'outbrain' || token === 'taboola'
224
+ }
225
+
226
+ function isRecommendationRail(element: Element): boolean {
227
+ const names = `${element.getAttribute('class') ?? ''} ${element.id}`.toLowerCase()
228
+ if (!names.split(/[\s_-]+/).some(isRailName)) return false
229
+ if (element.querySelectorAll('a').length < RAIL_MIN_LINKS) return false
230
+ const text = visibleTextLength(element)
231
+ return text > 0 && linkTextLength(element) / text >= RAIL_LINK_DENSITY
232
+ }
233
+
234
+ /** Marker attribute carried through Readability (it keeps `data-*`). */
235
+ const RAIL_ATTR = 'data-dsh-rail'
236
+ const RAIL_ATTR_TEXT = ` ${RAIL_ATTR}=""`
237
+
238
+ function isProse(block: Element): boolean {
239
+ const text = visibleTextLength(block)
240
+ return text >= RAIL_MIN_PROSE_CHARS && linkTextLength(block) / text < RAIL_PROSE_LINK_DENSITY
241
+ }
242
+
243
+ /**
244
+ * A copy of `document` whose recommendation rails ("post_recommends",
245
+ * "related-posts", Outbrain/Taboola widgets) carry a marker attribute, or
246
+ * null when it has none. Readability scores class/id names and text, never
247
+ * `data-*`, so it picks the same container on the copy as on the original —
248
+ * removing the rails beforehand instead was measured to shift its pick to a
249
+ * larger ancestor (WCXB 4206).
250
+ */
251
+ export function markRecommendationRails(document: Document): Document | null {
252
+ const body = document.body
253
+ if (body === null) return null
254
+ const rails: Element[] = []
255
+ const walk = (element: Element): void => {
256
+ for (const child of childElements(element)) {
257
+ if (isRecommendationRail(child)) rails.push(child)
258
+ else walk(child)
259
+ }
260
+ }
261
+ walk(body)
262
+ // Most pages have no rail: skip the clone. Otherwise mark the original
263
+ // just long enough to clone it, so the caller's document ends unchanged.
264
+ if (rails.length === 0) return null
265
+ for (const rail of rails) rail.setAttribute(RAIL_ATTR, '')
266
+ const copy = document.cloneNode(true) as Document
267
+ for (const rail of rails) rail.removeAttribute(RAIL_ATTR)
268
+ return copy
269
+ }
270
+
271
+ /**
272
+ * Split a Readability extraction of a rail-marked page into the extraction
273
+ * exactly as it would be unmarked, and — when a marked rail follows prose —
274
+ * the extraction without those rails.
275
+ *
276
+ * The trailing prune cannot catch a rail that outweighs the article (on a
277
+ * short 163.com post the rail is 86% of Readability's pick, past the share
278
+ * guards that keep docs indexes intact), and Readability strips the class
279
+ * names that would tell the two apart, hence the marker. A rail is removed
280
+ * only when it is recognised by name AND link density AND sits after some
281
+ * prose inside the extraction: a category page whose "related-product" grid
282
+ * opens the content (WCXB 4247) is the content, not a rail.
283
+ */
284
+ export function splitRails(articleHtml: string, document: Document): { plain: string, derailed: string | null } {
285
+ if (!articleHtml.includes(RAIL_ATTR)) return { plain: articleHtml, derailed: null }
286
+ const plain = articleHtml.split(RAIL_ATTR_TEXT).join('')
287
+ const container = htmlFragment(articleHtml, document)
288
+ const prose = Array.from(container.querySelectorAll('p, div, td, section, article')).filter(block => !block.hasAttribute(RAIL_ATTR) && block.closest(`[${RAIL_ATTR}]`) === null && isProse(block))
289
+ let removed = 0
290
+ for (const rail of Array.from(container.querySelectorAll(`[${RAIL_ATTR}]`))) {
291
+ // eslint-disable-next-line no-bitwise
292
+ const preceded = prose.some(block => (block.compareDocumentPosition(rail) & 4 /* FOLLOWING */) !== 0 && !block.contains(rail))
293
+ if (preceded) {
294
+ rail.remove()
295
+ removed++
296
+ }
297
+ }
298
+ if (removed === 0 || visibleTextLength(container) < RAIL_MIN_PROSE_CHARS) return { plain, derailed: null }
299
+ return { plain, derailed: container.innerHTML.split(RAIL_ATTR_TEXT).join('') }
300
+ }
301
+
302
+ // ------------------------------------------------------------ helpers ----
303
+
304
+ /**
305
+ * An element's element children as an array. Walks sibling pointers:
306
+ * jsdom's `HTMLCollection` is a proxy whose indexed access dominates
307
+ * profiles when `Array.from(el.children)` runs on every element of a large
308
+ * page (measured: 5.2s for a 4,000-paragraph article).
309
+ */
310
+ export function childElements(element: Element): Element[] {
311
+ const out: Element[] = []
312
+ for (let child = element.firstElementChild; child !== null; child = child.nextElementSibling) out.push(child)
313
+ return out
314
+ }
315
+
316
+ function htmlFragment(html: string, document: Document): Element {
317
+ const container = document.createElement('div')
318
+ container.innerHTML = html
319
+ return container
320
+ }
321
+
322
+ interface ReadabilityPick { content: string, title?: string }
323
+
324
+ /** Run Readability on a clone of `document`; null on failure/empty. */
325
+ function runReadability(document: Document, options?: ConstructorParameters<typeof Readability>[1]): ReadabilityPick | null {
326
+ try {
327
+ const cloned = document.cloneNode(true) as Document
328
+ const article = new Readability(cloned, options).parse()
329
+ if (article === null || typeof article.content !== 'string' || article.content === '') return null
330
+ return { content: article.content, title: article.title ?? undefined }
331
+ } catch {
332
+ return null
333
+ }
334
+ }
335
+
336
+ /**
337
+ * Rewrite every `<noscript>` into a `<div>` holding its markup.
338
+ *
339
+ * Script-less parsing (jsdom, static snapshots) exposes noscript content as
340
+ * markup that DOMPurify and Readability then delete wholesale — on Discourse
341
+ * that is the entire server-rendered thread, measured 16–36KB per page.
342
+ * Scripting-enabled parsing leaves it as one raw text node, which is
343
+ * reparsed here so both parser modes converge.
344
+ */
345
+ export function unwrapNoscripts(document: Document): number {
346
+ let count = 0
347
+ for (const noscript of Array.from(document.querySelectorAll('noscript'))) {
348
+ const div = document.createElement('div')
349
+ const onlyText = noscript.children.length === 0
350
+ if (onlyText) div.innerHTML = noscript.textContent ?? ''
351
+ else while (noscript.firstChild !== null) div.appendChild(noscript.firstChild)
352
+ noscript.replaceWith(div)
353
+ count++
354
+ }
355
+ return count
356
+ }
357
+
358
+ /** Visible text inside `<noscript>` elements (parsed either way). */
359
+ export function noscriptTextLength(document: Document): number {
360
+ let total = 0
361
+ for (const noscript of Array.from(document.querySelectorAll('noscript'))) {
362
+ if (noscript.children.length === 0) {
363
+ const div = document.createElement('div')
364
+ div.innerHTML = noscript.textContent ?? ''
365
+ total += visibleTextLength(div)
366
+ } else {
367
+ total += visibleTextLength(noscript, new Set(['script', 'style']))
368
+ }
369
+ }
370
+ return total
371
+ }
372
+
373
+ /** Semantic main-container selectors, most specific first. */
374
+ const MAIN_SELECTORS = [
375
+ 'main', '[role="main"]', '#main-content', '#maincontent', '#main', '.main-content',
376
+ '#content', '#primary', '.content',
377
+ ]
378
+
379
+ /** Find the page's semantic main container, if it carries real text. */
380
+ function findMainContainer(document: Document): Element | null {
381
+ const articles = document.querySelectorAll('article')
382
+ for (const selector of MAIN_SELECTORS) {
383
+ const element = document.querySelector(selector)
384
+ if (element !== null && visibleTextLength(element) >= 200) return element
385
+ }
386
+ if (articles.length === 1 && articles[0] !== undefined && visibleTextLength(articles[0]) >= 200) return articles[0]
387
+ return null
388
+ }
389
+
390
+ /** Sub-tokens of class/id names that mark page chrome, not content. */
391
+ const CHROME_TOKENS = new Set([
392
+ 'cookie', 'cookies', 'cookiebar', 'consent', 'gdpr', 'onetrust', 'cmp', 'ccpa',
393
+ 'newsletter', 'subscribe', 'popup', 'modal', 'overlay', 'lightbox', 'skip', 'skiplink',
394
+ 'breadcrumb', 'breadcrumbs', 'share', 'sharing', 'social', 'socials', 'advert',
395
+ 'advertisement', 'ads', 'ad', 'sponsor', 'sponsored', 'navbar', 'navigation', 'megamenu',
396
+ 'menu', 'sidebar', 'promo', 'banner', 'toast', 'drawer', 'offcanvas',
397
+ ])
398
+
399
+ const CHROME_ROLES = new Set(['navigation', 'dialog', 'alertdialog', 'complementary', 'contentinfo', 'search', 'banner'])
400
+
401
+ /**
402
+ * Cookie/consent banner wording (EN/DE/FR/ES/IT/NL/ZH), as a regex source
403
+ * shared with the selector's `consent_hits` feature so both read the same
404
+ * languages. Chinese banners say "Cookie" or 隐私; 同意 ("agree") alone is
405
+ * everyday prose and only counts as the banner's ACTION below.
406
+ */
407
+ export const CONSENT_TERMS = String.raw`\b(cookies?|consent|datenschutz\w*|einwilligung|zustimm\w*|akzeptier\w*|privacy|gdpr|rgpd)\b|隐私`
408
+ const CONSENT_TEXT = new RegExp(CONSENT_TERMS, 'i')
409
+ const CONSENT_ACTION = /\b(accept|agree|allow|reject|decline|manage|preferences|settings|akzeptieren|zustimmen|einverstanden|ablehnen|accepter|aceptar|accetta|accepteren)\b|接受|同意/i
410
+
411
+ function isChromeElement(element: Element): boolean {
412
+ const role = element.getAttribute('role')
413
+ if (role !== null && CHROME_ROLES.has(role.toLowerCase())) return true
414
+ if (element.hasAttribute('hidden') || element.getAttribute('aria-hidden') === 'true' || element.getAttribute('aria-modal') === 'true') return true
415
+ const style = element.getAttribute('style')
416
+ if (style !== null && /(display\s*:\s*none|visibility\s*:\s*hidden)/i.test(style)) return true
417
+ const names = `${element.getAttribute('class') ?? ''} ${element.id}`.toLowerCase()
418
+ if (names.trim() === '') return false
419
+ for (const token of names.split(/[\s_-]+/)) {
420
+ if (CHROME_TOKENS.has(token)) return true
421
+ }
422
+ return false
423
+ }
424
+
425
+ /**
426
+ * Strip page chrome from a cloned body: chrome-named/roled subtrees, hidden
427
+ * subtrees, and consent banners recognised by their wording. A subtree is
428
+ * only removed while it is a minority of the page, so a site that names its
429
+ * article wrapper `content-modal` cannot lose everything.
430
+ */
431
+ export function stripPageChrome(body: Element): void {
432
+ const total = Math.max(1, visibleTextLength(body))
433
+ const doomed: Element[] = []
434
+ const walk = (element: Element): void => {
435
+ for (const child of childElements(element)) {
436
+ if (isChromeElement(child) && visibleTextLength(child) <= 0.4 * total) {
437
+ doomed.push(child)
438
+ continue
439
+ }
440
+ walk(child)
441
+ }
442
+ }
443
+ walk(body)
444
+ for (const element of doomed) element.remove()
445
+ // Consent banners without telltale names: climb from the wording to the
446
+ // largest ancestor that is still banner-sized.
447
+ const ownerDocument = body.ownerDocument
448
+ const walker = ownerDocument.createTreeWalker(body, 4 /* SHOW_TEXT */)
449
+ const hits: Element[] = []
450
+ for (let node = walker.nextNode(); node !== null; node = walker.nextNode()) {
451
+ const text = node.nodeValue ?? ''
452
+ if (text.length < 20 || !CONSENT_TEXT.test(text)) continue
453
+ let block = node.parentElement
454
+ if (block === null) continue
455
+ const blockText = block.textContent ?? ''
456
+ if (!CONSENT_ACTION.test(blockText) && !CONSENT_ACTION.test(block.parentElement?.textContent ?? '')) continue
457
+ while (block.parentElement !== null && block.parentElement !== body && visibleTextLength(block.parentElement) < 1_200) {
458
+ block = block.parentElement
459
+ }
460
+ if (visibleTextLength(block) < 1_200) hits.push(block)
461
+ }
462
+ for (const element of hits) if (element.isConnected) element.remove()
463
+ }
464
+
465
+ /** Record-region discovery result. */
466
+ export interface RecordRegion {
467
+ items: Element[]
468
+ /** Total visible text of the items. */
469
+ text: number
470
+ }
471
+
472
+ /**
473
+ * Find the dominant repeated-record region: the sibling group with the same
474
+ * tag + class signature, at least four members, carrying the most text —
475
+ * product cards on a collection page, rows on a listing. Menu-like groups
476
+ * (tiny items, flat subtrees) are skipped.
477
+ */
478
+ export function findRecordRegion(body: Element): RecordRegion | null {
479
+ let best: RecordRegion | null = null
480
+ const visit = (element: Element): void => {
481
+ const name = element.nodeName.toLowerCase()
482
+ if (NOISE_SUBTREE_TAGS.has(name)) return
483
+ const children = childElements(element)
484
+ if (children.length >= 4) {
485
+ const groups = new Map<string, Element[]>()
486
+ for (const child of children) {
487
+ const classes = (child.getAttribute('class') ?? '').replace(/\d+/g, '').trim().split(/\s+/).filter(Boolean).sort().slice(0, 2).join('.')
488
+ const signature = `${child.nodeName}.${classes}`
489
+ const group = groups.get(signature)
490
+ if (group === undefined) groups.set(signature, [child])
491
+ else group.push(child)
492
+ }
493
+ for (const items of groups.values()) {
494
+ if (items.length < 4) continue
495
+ let text = 0
496
+ let depth = 0
497
+ for (const item of items) {
498
+ text += visibleTextLength(item)
499
+ depth += item.getElementsByTagName('*').length
500
+ }
501
+ if (text / items.length < 15 || depth / items.length < 2) continue
502
+ if (best === null || text > best.text) best = { items, text }
503
+ }
504
+ }
505
+ for (const child of children) visit(child)
506
+ }
507
+ visit(body)
508
+ return best
509
+ }
510
+
511
+ /**
512
+ * A post's timestamp as a locale-neutral calendar date (`2024-09-05`, UTC).
513
+ * Deliberately not any UI's localized rendering: the reader may be any
514
+ * language, and the page itself prints dates in its own locale.
515
+ */
516
+ function formatForumDate(iso: unknown): string {
517
+ if (typeof iso !== 'string') return ''
518
+ const date = new Date(iso)
519
+ if (Number.isNaN(date.getTime())) return ''
520
+ return date.toISOString().slice(0, 10)
521
+ }
522
+
523
+ /**
524
+ * Discourse ships the topic's post stream as JSON in
525
+ * `#data-preloaded[data-preloaded]` (a map of JSON strings). Rebuild the
526
+ * thread as HTML: author, date, cooked post body.
527
+ */
528
+ export function discoursePreloadedHtml(document: Document): string | null {
529
+ const holder = document.querySelector('#data-preloaded[data-preloaded]')
530
+ const raw = holder?.getAttribute('data-preloaded')
531
+ if (raw == null) return null
532
+ try {
533
+ const outer = JSON.parse(raw) as Record<string, unknown>
534
+ for (const [key, value] of Object.entries(outer)) {
535
+ if (!key.startsWith('topic_') || typeof value !== 'string') continue
536
+ const topic = JSON.parse(value) as { title?: string, post_stream?: { posts?: { username?: string, created_at?: string, cooked?: string }[] } }
537
+ const posts = topic.post_stream?.posts ?? []
538
+ if (posts.length === 0) continue
539
+ const parts = posts.map(post => `<div><p>${escapeHtml(post.username ?? '')}</p><p>${escapeHtml(formatForumDate(post.created_at))}</p>${post.cooked ?? ''}</div>`)
540
+ return parts.join('\n')
541
+ }
542
+ } catch {
543
+ return null
544
+ }
545
+ return null
546
+ }
547
+
548
+ function escapeHtml(text: string): string {
549
+ return text.replace(/[&<>"]/g, ch => ({ '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;' })[ch] ?? ch)
550
+ }
551
+
552
+ // ---------------------------------------------------------- generation ----
553
+
554
+ /** Everything candidate generation learned about the page, for features. */
555
+ export interface GenerationFacts {
556
+ readabilityNull: boolean
557
+ gatePass: boolean
558
+ pruneChanged: boolean
559
+ noscriptText: number
560
+ record: RecordRegion | null
561
+ hasMain: boolean
562
+ hasPreloaded: boolean
563
+ /** The semantic main container, when found. */
564
+ main: Element | null
565
+ }
566
+
567
+ /** Below this much noscript text the noscript-unwrapped paths are skipped. */
568
+ const NOSCRIPT_MIN_TEXT_CHARS = 200
569
+
570
+ /**
571
+ * Propose every candidate extraction for one parsed page.
572
+ * @param document - the parsed page (data URIs already elided); not mutated.
573
+ */
574
+ export function generateCandidates(document: Document): { candidates: RawCandidate[], facts: GenerationFacts } {
575
+ const candidates: RawCandidate[] = []
576
+ const body = document.body
577
+ const bodyHtml = body?.innerHTML ?? ''
578
+
579
+ // Rails are marked before Readability (it drops the class names that
580
+ // identify them) and cut from its output; the gate and `artu` see the
581
+ // extraction exactly as unmarked, so the legacy marker and the "keep
582
+ // everything" alternative are unchanged.
583
+ const marked = runReadability(markRecommendationRails(document) ?? document)
584
+ const rails = marked === null ? null : splitRails(marked.content, document)
585
+ const art = marked === null || rails === null ? null : { ...marked, content: rails.plain }
586
+ const gatePass = art !== null && articleCoversPage(art.content, document)
587
+ let pruneChanged = false
588
+ if (art !== null) {
589
+ const pruned = prunedArticleContent(rails?.derailed ?? art.content, document)
590
+ pruneChanged = pruned !== art.content
591
+ candidates.push({ kind: 'art', html: pruned, title: art.title, legacy: gatePass })
592
+ if (pruneChanged) candidates.push({ kind: 'artu', html: art.content, title: art.title, legacy: false })
593
+ }
594
+ // `linkDensityModifier` is a public Readability option missing from the
595
+ // bundled type declarations; it relaxes the link-density cleaning cutoffs.
596
+ const lenient = runReadability(document, { linkDensityModifier: 0.4 } as ConstructorParameters<typeof Readability>[1])
597
+ if (lenient !== null && lenient.content !== art?.content) {
598
+ candidates.push({ kind: 'artx', html: lenient.content, title: lenient.title, legacy: false })
599
+ }
600
+ const title = art?.title ?? lenient?.title
601
+
602
+ const main = body === null ? null : findMainContainer(document)
603
+ if (main !== null) candidates.push({ kind: 'main', html: main.innerHTML, title, legacy: false })
604
+
605
+ candidates.push({ kind: 'doc', html: bodyHtml, legacy: !gatePass })
606
+
607
+ if (body !== null) {
608
+ const cleaned = body.cloneNode(true) as Element
609
+ stripPageChrome(cleaned)
610
+ candidates.push({ kind: 'docc', html: cleaned.innerHTML, legacy: false })
611
+ }
612
+
613
+ const record = body === null ? null : findRecordRegion(body)
614
+ if (record !== null) {
615
+ const heading = document.querySelector('h1')
616
+ const html = (heading !== null ? heading.outerHTML : '') + record.items.map(item => item.outerHTML).join('\n')
617
+ candidates.push({ kind: 'rec', html, legacy: false })
618
+ }
619
+
620
+ const noscriptText = noscriptTextLength(document)
621
+ if (noscriptText >= NOSCRIPT_MIN_TEXT_CHARS) {
622
+ const unwrapped = document.cloneNode(true) as Document
623
+ unwrapNoscripts(unwrapped)
624
+ const nsArt = runReadability(unwrapped)
625
+ if (nsArt !== null) candidates.push({ kind: 'ns_art', html: prunedArticleContent(nsArt.content, unwrapped), title: nsArt.title, legacy: false })
626
+ candidates.push({ kind: 'ns_doc', html: unwrapped.body?.innerHTML ?? '', legacy: false })
627
+ }
628
+
629
+ const preloaded = discoursePreloadedHtml(document)
630
+ if (preloaded !== null) candidates.push({ kind: 'pre', html: preloaded, title, legacy: false })
631
+
632
+ // Identical proposals (a `<main>` that IS Readability's pick, a chrome
633
+ // strip that removed nothing) would only cost another sanitize pass.
634
+ const seen = new Set(candidates.filter(candidate => candidate.legacy).map(candidate => candidate.html))
635
+ const unique = candidates.filter(candidate => {
636
+ if (candidate.legacy) return true
637
+ if (seen.has(candidate.html)) return false
638
+ seen.add(candidate.html)
639
+ return true
640
+ })
641
+
642
+ return {
643
+ candidates: unique,
644
+ facts: {
645
+ readabilityNull: art === null,
646
+ gatePass,
647
+ pruneChanged,
648
+ noscriptText,
649
+ record,
650
+ hasMain: main !== null,
651
+ hasPreloaded: preloaded !== null,
652
+ main,
653
+ },
654
+ }
655
+ }
@@ -177,32 +177,6 @@ export function PlaywrightCard(props: PlaywrightCardProps) {
177
177
  onEdit={(text) => { props.edit('maxConcurrency', text) }}
178
178
  onReset={() => { props.resetField('maxConcurrency') }}
179
179
  />
180
- <ValueField
181
- id="plugin-config-playwright-challenge-wait"
182
- label={t('challengeWaitMs')}
183
- hint={t('challengeWaitMsHint')}
184
- placeholder={t('challengeWaitMsPlaceholder')}
185
- overriddenLabel={t('overridden')}
186
- resetLabel={t('reset')}
187
- invalidLabel={t('invalidText')}
188
- disabled={disabled}
189
- {...state.challengeWaitMs}
190
- onEdit={(text) => { props.edit('challengeWaitMs', text) }}
191
- onReset={() => { props.resetField('challengeWaitMs') }}
192
- />
193
- <ValueField
194
- id="plugin-config-playwright-scroll-budget"
195
- label={t('scrollBudgetMs')}
196
- hint={t('scrollBudgetMsHint')}
197
- placeholder={t('scrollBudgetMsPlaceholder')}
198
- overriddenLabel={t('overridden')}
199
- resetLabel={t('reset')}
200
- invalidLabel={t('invalidText')}
201
- disabled={disabled}
202
- {...state.scrollBudgetMs}
203
- onEdit={(text) => { props.edit('scrollBudgetMs', text) }}
204
- onReset={() => { props.resetField('scrollBudgetMs') }}
205
- />
206
180
  </CardForm>
207
181
  )
208
182
  }