dsh-web-fetch-playwright 0.2.19 → 0.2.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,639 @@
1
+ /**
2
+ * Candidate generation for the denoise pipeline.
3
+ *
4
+ * The pipeline used to be ONE chain of hard decisions (Readability → coverage
5
+ * gate → trailing prune → whole-document fallback), each tuned by a global
6
+ * threshold. Every page shape that disagrees with a threshold's notion of
7
+ * "content" (link-dense docs indexes vs. recommendation rails, noscript
8
+ * server fallbacks on forums vs. lazy-image wrappers elsewhere) turned each
9
+ * fix into a regression somewhere else. Here every heuristic instead
10
+ * *proposes* a candidate extraction, and a learned selector
11
+ * (`./selector.ts`) picks one per page from page-level and candidate-level
12
+ * features — adding a heuristic can only add an option, never silently
13
+ * override the others.
14
+ *
15
+ * Candidates (see {@link CandidateKind}):
16
+ * - `art` Readability, trailing link farms pruned (the 0.2.18 article path)
17
+ * - `artu` Readability unpruned (link directories that ARE content)
18
+ * - `artx` Readability with a lenient link-density modifier
19
+ * - `main` the page's semantic main container (`<main>`, `[role=main]`, …)
20
+ * - `doc` the whole sanitized document (the 0.2.18 fallback)
21
+ * - `docc` the whole document minus consent/cookie banners, hidden and
22
+ * chrome-classed subtrees
23
+ * - `rec` the dominant repeated-record region (product cards, list rows)
24
+ * - `ns_art` / `ns_doc` the same two paths after unwrapping `<noscript>`
25
+ * server fallbacks (Discourse and friends ship their whole thread
26
+ * there for script-less clients)
27
+ * - `pre` Discourse `data-preloaded` post stream
28
+ * - `blk` blocks kept by the block classifier (`./blocks.ts`), added by
29
+ * the orchestrator once page features exist
30
+ *
31
+ * @module dsh-web-fetch-playwright/candidates
32
+ */
33
+
34
+ import { Readability } from '@mozilla/readability'
35
+
36
+ /** Every candidate extraction strategy, in the selector's one-hot order. */
37
+ export const CANDIDATE_KINDS = ['art', 'artu', 'artx', 'main', 'doc', 'docc', 'rec', 'ns_art', 'ns_doc', 'pre', 'blk'] as const
38
+ export type CandidateKind = typeof CANDIDATE_KINDS[number]
39
+
40
+ /** One proposed extraction, still as (unsanitized) HTML. */
41
+ export interface RawCandidate {
42
+ kind: CandidateKind
43
+ /** HTML of the extraction; sanitized by the caller. */
44
+ html: string
45
+ /** Title to prepend when the body does not open with a heading. */
46
+ title?: string
47
+ /** Whether the 0.2.18 pipeline would have returned exactly this candidate. */
48
+ legacy: boolean
49
+ }
50
+
51
+ /** Layout/noise tags DOMPurify removes outright (with their content). */
52
+ export const FORBID_TAGS = [
53
+ 'nav', 'aside', 'header', 'footer', 'form', 'svg', 'iframe', 'noscript',
54
+ 'button', 'select', 'option', 'input', 'textarea', 'dialog', 'canvas',
55
+ 'video', 'audio', 'template',
56
+ ]
57
+
58
+ /** Tags whose subtrees never count as visible text. */
59
+ export const NOISE_SUBTREE_TAGS = new Set([...FORBID_TAGS, 'script', 'style'])
60
+
61
+ // ---------------------------------------------------------------- text ----
62
+
63
+ /**
64
+ * Count non-whitespace characters under `root`, skipping noise subtrees.
65
+ * Whitespace is ignored so CJK and latin pages measure comparable densities.
66
+ */
67
+ export function visibleTextLength(root: Node | null, skip: Set<string> = NOISE_SUBTREE_TAGS): number {
68
+ if (root === null) return 0
69
+ let length = 0
70
+ const visit = (node: Node): void => {
71
+ if (node.nodeType === 3) {
72
+ length += (node.nodeValue ?? '').replace(/\s+/g, '').length
73
+ return
74
+ }
75
+ if (node.nodeType !== 1 || skip.has(node.nodeName.toLowerCase())) return
76
+ for (let child = node.firstChild; child !== null; child = child.nextSibling) visit(child)
77
+ }
78
+ visit(root)
79
+ return length
80
+ }
81
+
82
+ /** Count non-whitespace characters under `root` that sit inside an anchor. */
83
+ export function linkTextLength(root: Node | null): number {
84
+ if (root === null) return 0
85
+ let length = 0
86
+ const visit = (node: Node, insideAnchor: boolean): void => {
87
+ if (node.nodeType === 3) {
88
+ if (insideAnchor) length += (node.nodeValue ?? '').replace(/\s+/g, '').length
89
+ return
90
+ }
91
+ if (node.nodeType !== 1 || NOISE_SUBTREE_TAGS.has(node.nodeName.toLowerCase())) return
92
+ const anchor = insideAnchor || node.nodeName.toLowerCase() === 'a'
93
+ for (let child = node.firstChild; child !== null; child = child.nextSibling) visit(child, anchor)
94
+ }
95
+ visit(root, false)
96
+ return length
97
+ }
98
+
99
+ // ------------------------------------------------------- coverage gate ----
100
+
101
+ /** Below this much visible page text the gate trusts any extraction. */
102
+ const GATE_MIN_BODY_TEXT_CHARS = 1_000
103
+ /** Minimum characters a trusted Readability extraction carries. */
104
+ export const GATE_MIN_ARTICLE_TEXT_CHARS = 250
105
+ /** Minimum share of the page's visible text a trusted extraction covers. */
106
+ const GATE_MIN_ARTICLE_COVERAGE = 0.1
107
+
108
+ /**
109
+ * The 0.2.18 acceptance gate: distrust an extraction that is tiny in absolute
110
+ * terms or covers a sliver of the page's visible text. Kept as a selector
111
+ * feature and as the legacy-choice marker; it no longer decides alone.
112
+ */
113
+ export function articleCoversPage(articleHtml: string, document: Document): boolean {
114
+ const bodyText = visibleTextLength(document.body)
115
+ if (bodyText < GATE_MIN_BODY_TEXT_CHARS) return true
116
+ const container = document.createElement('div')
117
+ container.innerHTML = articleHtml
118
+ const articleText = visibleTextLength(container)
119
+ return articleText >= GATE_MIN_ARTICLE_TEXT_CHARS
120
+ && articleText / bodyText >= GATE_MIN_ARTICLE_COVERAGE
121
+ }
122
+
123
+ // ------------------------------------------------------ trailing prune ----
124
+
125
+ const TRAILING_MIN_TEXT_CHARS = 40
126
+ const TRAILING_LINK_DENSITY = 0.4
127
+ const TRAILING_LINK_COUNT = 3
128
+ const TRAILING_LINK_TEXT_CHARS = 60
129
+ const TRAILING_MAX_ARTICLE_SHARE = 0.6
130
+ const TRAILING_MAX_RUN_SHARE = 0.6
131
+ const TRAILING_UNIFORM_DENSITY_BAND = 0.25
132
+
133
+ /**
134
+ * Prune trailing link-dense sections (recommendation rails, share bars) from
135
+ * a Readability extraction — the 0.2.18 heuristic (see CHANGELOG 0.2.18 for
136
+ * the 163.com badcase and the Hacker News guards). Since 0.2.20 its output is
137
+ * one candidate (`art`) beside the unpruned extraction (`artu`), so pages
138
+ * whose trailing link block IS content (docs indexes, contributor lists) are
139
+ * no longer forced through it.
140
+ */
141
+ function pruneTrailingLinkDenseSections(container: Element): void {
142
+ const articleText = visibleTextLength(container)
143
+ if (articleText === 0) return
144
+ for (const element of [container, ...Array.from(container.querySelectorAll('*'))]) {
145
+ if (!container.contains(element) && element !== container) continue
146
+ removeTrailingLinkDenseRun(element, articleText)
147
+ }
148
+ }
149
+
150
+ function removeTrailingLinkDenseRun(parent: Element, articleText: number): void {
151
+ const children = childElements(parent)
152
+ if (children.length === 0) return
153
+ const containerText = visibleTextLength(parent)
154
+ if (containerText === 0) return
155
+ const run: Element[] = []
156
+ let runText = 0
157
+ let runLinked = 0
158
+ let stopIndex = -1
159
+ for (let index = children.length - 1; index >= 0; index--) {
160
+ const child = children[index]
161
+ if (child === undefined) break
162
+ const text = visibleTextLength(child)
163
+ if (text < TRAILING_MIN_TEXT_CHARS) {
164
+ run.push(child)
165
+ continue
166
+ }
167
+ const linked = linkTextLength(child)
168
+ const linkCount = child.querySelectorAll('a').length
169
+ if (
170
+ linked / text >= TRAILING_LINK_DENSITY
171
+ && (linkCount >= TRAILING_LINK_COUNT || linked >= TRAILING_LINK_TEXT_CHARS)
172
+ && text <= TRAILING_MAX_ARTICLE_SHARE * articleText
173
+ ) {
174
+ run.push(child)
175
+ runText += text
176
+ runLinked += linked
177
+ continue
178
+ }
179
+ stopIndex = index
180
+ break
181
+ }
182
+ if (runText === 0 || stopIndex === -1) return
183
+ const stopChild = children[stopIndex]
184
+ if (stopChild === undefined) return
185
+ const stopDensity = linkTextLength(stopChild) / Math.max(1, visibleTextLength(stopChild))
186
+ if (Math.abs(runLinked / runText - stopDensity) <= TRAILING_UNIFORM_DENSITY_BAND) return
187
+ if (runText > TRAILING_MAX_RUN_SHARE * containerText) return
188
+ for (const child of run) child.remove()
189
+ }
190
+
191
+ /** Prune with the 0.2.18 misfire guard; returns the original on a misfire. */
192
+ export function prunedArticleContent(articleHtml: string, document: Document): string {
193
+ const container = document.createElement('div')
194
+ container.innerHTML = articleHtml
195
+ const before = visibleTextLength(container)
196
+ if (before === 0) return articleHtml
197
+ pruneTrailingLinkDenseSections(container)
198
+ const after = visibleTextLength(container)
199
+ if (after === 0 || (before >= GATE_MIN_ARTICLE_TEXT_CHARS && after < GATE_MIN_ARTICLE_TEXT_CHARS)) {
200
+ return articleHtml
201
+ }
202
+ return container.innerHTML
203
+ }
204
+
205
+ // ------------------------------------------------- recommendation rails ----
206
+
207
+ /** Link density at or above which a recommendation-named subtree is a rail. */
208
+ const RAIL_LINK_DENSITY = 0.3
209
+ const RAIL_MIN_LINKS = 4
210
+ /** Prose (low-link text) the page must keep outside the rails. */
211
+ const RAIL_MIN_PROSE_CHARS = 80
212
+ const RAIL_PROSE_LINK_DENSITY = 0.2
213
+
214
+ /** Class/id sub-tokens naming a recommendation / related-content rail. */
215
+ function isRailName(token: string): boolean {
216
+ return token.includes('recommend') || token.startsWith('related')
217
+ || token === 'outbrain' || token === 'taboola' || token === 'tuijian' || token === 'xgtj'
218
+ }
219
+
220
+ function isRecommendationRail(element: Element): boolean {
221
+ const names = `${element.getAttribute('class') ?? ''} ${element.id}`.toLowerCase()
222
+ if (!names.split(/[\s_-]+/).some(isRailName)) return false
223
+ if (element.querySelectorAll('a').length < RAIL_MIN_LINKS) return false
224
+ const text = visibleTextLength(element)
225
+ return text > 0 && linkTextLength(element) / text >= RAIL_LINK_DENSITY
226
+ }
227
+
228
+ /** Marker attribute carried through Readability (it keeps `data-*`). */
229
+ const RAIL_ATTR = 'data-dsh-rail'
230
+ const RAIL_ATTR_TEXT = ` ${RAIL_ATTR}=""`
231
+
232
+ function isProse(block: Element): boolean {
233
+ const text = visibleTextLength(block)
234
+ return text >= RAIL_MIN_PROSE_CHARS && linkTextLength(block) / text < RAIL_PROSE_LINK_DENSITY
235
+ }
236
+
237
+ /**
238
+ * A copy of `document` whose recommendation rails ("post_recommends",
239
+ * "related-posts", Outbrain/Taboola widgets) carry a marker attribute, or
240
+ * null when it has none. Readability scores class/id names and text, never
241
+ * `data-*`, so it picks the same container on the copy as on the original —
242
+ * removing the rails beforehand instead was measured to shift its pick to a
243
+ * larger ancestor (WCXB 4206).
244
+ */
245
+ export function markRecommendationRails(document: Document): Document | null {
246
+ const body = document.body
247
+ if (body === null) return null
248
+ const rails: Element[] = []
249
+ const walk = (element: Element): void => {
250
+ for (const child of childElements(element)) {
251
+ if (isRecommendationRail(child)) rails.push(child)
252
+ else walk(child)
253
+ }
254
+ }
255
+ walk(body)
256
+ // Most pages have no rail: skip the clone. Otherwise mark the original
257
+ // just long enough to clone it, so the caller's document ends unchanged.
258
+ if (rails.length === 0) return null
259
+ for (const rail of rails) rail.setAttribute(RAIL_ATTR, '')
260
+ const copy = document.cloneNode(true) as Document
261
+ for (const rail of rails) rail.removeAttribute(RAIL_ATTR)
262
+ return copy
263
+ }
264
+
265
+ /**
266
+ * Split a Readability extraction of a rail-marked page into the extraction
267
+ * exactly as it would be unmarked, and — when a marked rail follows prose —
268
+ * the extraction without those rails.
269
+ *
270
+ * The trailing prune cannot catch a rail that outweighs the article (on a
271
+ * short 163.com post the rail is 86% of Readability's pick, past the share
272
+ * guards that keep docs indexes intact), and Readability strips the class
273
+ * names that would tell the two apart, hence the marker. A rail is removed
274
+ * only when it is recognised by name AND link density AND sits after some
275
+ * prose inside the extraction: a category page whose "related-product" grid
276
+ * opens the content (WCXB 4247) is the content, not a rail.
277
+ */
278
+ export function splitRails(articleHtml: string, document: Document): { plain: string, derailed: string | null } {
279
+ if (!articleHtml.includes(RAIL_ATTR)) return { plain: articleHtml, derailed: null }
280
+ const plain = articleHtml.split(RAIL_ATTR_TEXT).join('')
281
+ const container = htmlFragment(articleHtml, document)
282
+ const prose = Array.from(container.querySelectorAll('p, div, td, section, article')).filter(block => !block.hasAttribute(RAIL_ATTR) && block.closest(`[${RAIL_ATTR}]`) === null && isProse(block))
283
+ let removed = 0
284
+ for (const rail of Array.from(container.querySelectorAll(`[${RAIL_ATTR}]`))) {
285
+ // eslint-disable-next-line no-bitwise
286
+ const preceded = prose.some(block => (block.compareDocumentPosition(rail) & 4 /* FOLLOWING */) !== 0 && !block.contains(rail))
287
+ if (preceded) {
288
+ rail.remove()
289
+ removed++
290
+ }
291
+ }
292
+ if (removed === 0 || visibleTextLength(container) < RAIL_MIN_PROSE_CHARS) return { plain, derailed: null }
293
+ return { plain, derailed: container.innerHTML.split(RAIL_ATTR_TEXT).join('') }
294
+ }
295
+
296
+ // ------------------------------------------------------------ helpers ----
297
+
298
+ /**
299
+ * An element's element children as an array. Walks sibling pointers:
300
+ * jsdom's `HTMLCollection` is a proxy whose indexed access dominates
301
+ * profiles when `Array.from(el.children)` runs on every element of a large
302
+ * page (measured: 5.2s for a 4,000-paragraph article).
303
+ */
304
+ export function childElements(element: Element): Element[] {
305
+ const out: Element[] = []
306
+ for (let child = element.firstElementChild; child !== null; child = child.nextElementSibling) out.push(child)
307
+ return out
308
+ }
309
+
310
+ function htmlFragment(html: string, document: Document): Element {
311
+ const container = document.createElement('div')
312
+ container.innerHTML = html
313
+ return container
314
+ }
315
+
316
+ interface ReadabilityPick { content: string, title?: string }
317
+
318
+ /** Run Readability on a clone of `document`; null on failure/empty. */
319
+ function runReadability(document: Document, options?: ConstructorParameters<typeof Readability>[1]): ReadabilityPick | null {
320
+ try {
321
+ const cloned = document.cloneNode(true) as Document
322
+ const article = new Readability(cloned, options).parse()
323
+ if (article === null || typeof article.content !== 'string' || article.content === '') return null
324
+ return { content: article.content, title: article.title ?? undefined }
325
+ } catch {
326
+ return null
327
+ }
328
+ }
329
+
330
+ /**
331
+ * Rewrite every `<noscript>` into a `<div>` holding its markup.
332
+ *
333
+ * Script-less parsing (jsdom, static snapshots) exposes noscript content as
334
+ * markup that DOMPurify and Readability then delete wholesale — on Discourse
335
+ * that is the entire server-rendered thread, measured 16–36KB per page.
336
+ * Scripting-enabled parsing leaves it as one raw text node, which is
337
+ * reparsed here so both parser modes converge.
338
+ */
339
+ export function unwrapNoscripts(document: Document): number {
340
+ let count = 0
341
+ for (const noscript of Array.from(document.querySelectorAll('noscript'))) {
342
+ const div = document.createElement('div')
343
+ const onlyText = noscript.children.length === 0
344
+ if (onlyText) div.innerHTML = noscript.textContent ?? ''
345
+ else while (noscript.firstChild !== null) div.appendChild(noscript.firstChild)
346
+ noscript.replaceWith(div)
347
+ count++
348
+ }
349
+ return count
350
+ }
351
+
352
+ /** Visible text inside `<noscript>` elements (parsed either way). */
353
+ export function noscriptTextLength(document: Document): number {
354
+ let total = 0
355
+ for (const noscript of Array.from(document.querySelectorAll('noscript'))) {
356
+ if (noscript.children.length === 0) {
357
+ const div = document.createElement('div')
358
+ div.innerHTML = noscript.textContent ?? ''
359
+ total += visibleTextLength(div)
360
+ } else {
361
+ total += visibleTextLength(noscript, new Set(['script', 'style']))
362
+ }
363
+ }
364
+ return total
365
+ }
366
+
367
+ /** Semantic main-container selectors, most specific first. */
368
+ const MAIN_SELECTORS = [
369
+ 'main', '[role="main"]', '#main-content', '#maincontent', '#main', '.main-content',
370
+ '#content', '#primary', '.content',
371
+ ]
372
+
373
+ /** Find the page's semantic main container, if it carries real text. */
374
+ function findMainContainer(document: Document): Element | null {
375
+ const articles = document.querySelectorAll('article')
376
+ for (const selector of MAIN_SELECTORS) {
377
+ const element = document.querySelector(selector)
378
+ if (element !== null && visibleTextLength(element) >= 200) return element
379
+ }
380
+ if (articles.length === 1 && articles[0] !== undefined && visibleTextLength(articles[0]) >= 200) return articles[0]
381
+ return null
382
+ }
383
+
384
+ /** Sub-tokens of class/id names that mark page chrome, not content. */
385
+ const CHROME_TOKENS = new Set([
386
+ 'cookie', 'cookies', 'cookiebar', 'consent', 'gdpr', 'onetrust', 'cmp', 'ccpa',
387
+ 'newsletter', 'subscribe', 'popup', 'modal', 'overlay', 'lightbox', 'skip', 'skiplink',
388
+ 'breadcrumb', 'breadcrumbs', 'share', 'sharing', 'social', 'socials', 'advert',
389
+ 'advertisement', 'ads', 'ad', 'sponsor', 'sponsored', 'navbar', 'navigation', 'megamenu',
390
+ 'menu', 'sidebar', 'promo', 'banner', 'toast', 'drawer', 'offcanvas',
391
+ ])
392
+
393
+ const CHROME_ROLES = new Set(['navigation', 'dialog', 'alertdialog', 'complementary', 'contentinfo', 'search', 'banner'])
394
+
395
+ /** Cookie/consent banner wording (EN/DE/FR/ES/IT/NL/ZH). */
396
+ const CONSENT_TEXT = /\b(cookies?|consent|datenschutz\w*|einwilligung|zustimm\w*|akzeptier\w*|privacy|gdpr|rgpd)\b|隐私|同意/i
397
+ const CONSENT_ACTION = /\b(accept|agree|allow|reject|decline|manage|preferences|settings|akzeptieren|zustimmen|einverstanden|ablehnen|accepter|aceptar|accetta|accepteren)\b|接受|同意/i
398
+
399
+ function isChromeElement(element: Element): boolean {
400
+ const role = element.getAttribute('role')
401
+ if (role !== null && CHROME_ROLES.has(role.toLowerCase())) return true
402
+ if (element.hasAttribute('hidden') || element.getAttribute('aria-hidden') === 'true' || element.getAttribute('aria-modal') === 'true') return true
403
+ const style = element.getAttribute('style')
404
+ if (style !== null && /(display\s*:\s*none|visibility\s*:\s*hidden)/i.test(style)) return true
405
+ const names = `${element.getAttribute('class') ?? ''} ${element.id}`.toLowerCase()
406
+ if (names.trim() === '') return false
407
+ for (const token of names.split(/[\s_-]+/)) {
408
+ if (CHROME_TOKENS.has(token)) return true
409
+ }
410
+ return false
411
+ }
412
+
413
+ /**
414
+ * Strip page chrome from a cloned body: chrome-named/roled subtrees, hidden
415
+ * subtrees, and consent banners recognised by their wording. A subtree is
416
+ * only removed while it is a minority of the page, so a site that names its
417
+ * article wrapper `content-modal` cannot lose everything.
418
+ */
419
+ export function stripPageChrome(body: Element): void {
420
+ const total = Math.max(1, visibleTextLength(body))
421
+ const doomed: Element[] = []
422
+ const walk = (element: Element): void => {
423
+ for (const child of childElements(element)) {
424
+ if (isChromeElement(child) && visibleTextLength(child) <= 0.4 * total) {
425
+ doomed.push(child)
426
+ continue
427
+ }
428
+ walk(child)
429
+ }
430
+ }
431
+ walk(body)
432
+ for (const element of doomed) element.remove()
433
+ // Consent banners without telltale names: climb from the wording to the
434
+ // largest ancestor that is still banner-sized.
435
+ const ownerDocument = body.ownerDocument
436
+ const walker = ownerDocument.createTreeWalker(body, 4 /* SHOW_TEXT */)
437
+ const hits: Element[] = []
438
+ for (let node = walker.nextNode(); node !== null; node = walker.nextNode()) {
439
+ const text = node.nodeValue ?? ''
440
+ if (text.length < 20 || !CONSENT_TEXT.test(text)) continue
441
+ let block = node.parentElement
442
+ if (block === null) continue
443
+ const blockText = block.textContent ?? ''
444
+ if (!CONSENT_ACTION.test(blockText) && !CONSENT_ACTION.test(block.parentElement?.textContent ?? '')) continue
445
+ while (block.parentElement !== null && block.parentElement !== body && visibleTextLength(block.parentElement) < 1_200) {
446
+ block = block.parentElement
447
+ }
448
+ if (visibleTextLength(block) < 1_200) hits.push(block)
449
+ }
450
+ for (const element of hits) if (element.isConnected) element.remove()
451
+ }
452
+
453
+ /** Record-region discovery result. */
454
+ export interface RecordRegion {
455
+ items: Element[]
456
+ /** Total visible text of the items. */
457
+ text: number
458
+ }
459
+
460
+ /**
461
+ * Find the dominant repeated-record region: the sibling group with the same
462
+ * tag + class signature, at least four members, carrying the most text —
463
+ * product cards on a collection page, rows on a listing. Menu-like groups
464
+ * (tiny items, flat subtrees) are skipped.
465
+ */
466
+ export function findRecordRegion(body: Element): RecordRegion | null {
467
+ let best: RecordRegion | null = null
468
+ const visit = (element: Element): void => {
469
+ const name = element.nodeName.toLowerCase()
470
+ if (NOISE_SUBTREE_TAGS.has(name)) return
471
+ const children = childElements(element)
472
+ if (children.length >= 4) {
473
+ const groups = new Map<string, Element[]>()
474
+ for (const child of children) {
475
+ const classes = (child.getAttribute('class') ?? '').replace(/\d+/g, '').trim().split(/\s+/).filter(Boolean).sort().slice(0, 2).join('.')
476
+ const signature = `${child.nodeName}.${classes}`
477
+ const group = groups.get(signature)
478
+ if (group === undefined) groups.set(signature, [child])
479
+ else group.push(child)
480
+ }
481
+ for (const items of groups.values()) {
482
+ if (items.length < 4) continue
483
+ let text = 0
484
+ let depth = 0
485
+ for (const item of items) {
486
+ text += visibleTextLength(item)
487
+ depth += item.getElementsByTagName('*').length
488
+ }
489
+ if (text / items.length < 15 || depth / items.length < 2) continue
490
+ if (best === null || text > best.text) best = { items, text }
491
+ }
492
+ }
493
+ for (const child of children) visit(child)
494
+ }
495
+ visit(body)
496
+ return best
497
+ }
498
+
499
+ /** Format an ISO timestamp the way forum UIs print it ("September 5, 2024"). */
500
+ function formatForumDate(iso: unknown): string {
501
+ if (typeof iso !== 'string') return ''
502
+ const date = new Date(iso)
503
+ if (Number.isNaN(date.getTime())) return ''
504
+ return date.toLocaleDateString('en-US', { year: 'numeric', month: 'long', day: 'numeric', timeZone: 'UTC' })
505
+ }
506
+
507
+ /**
508
+ * Discourse ships the topic's post stream as JSON in
509
+ * `#data-preloaded[data-preloaded]` (a map of JSON strings). Rebuild the
510
+ * thread as HTML: author, date, cooked post body.
511
+ */
512
+ export function discoursePreloadedHtml(document: Document): string | null {
513
+ const holder = document.querySelector('#data-preloaded[data-preloaded]')
514
+ const raw = holder?.getAttribute('data-preloaded')
515
+ if (raw == null) return null
516
+ try {
517
+ const outer = JSON.parse(raw) as Record<string, unknown>
518
+ for (const [key, value] of Object.entries(outer)) {
519
+ if (!key.startsWith('topic_') || typeof value !== 'string') continue
520
+ const topic = JSON.parse(value) as { title?: string, post_stream?: { posts?: { username?: string, created_at?: string, cooked?: string }[] } }
521
+ const posts = topic.post_stream?.posts ?? []
522
+ if (posts.length === 0) continue
523
+ const parts = posts.map(post => `<div><p>${escapeHtml(post.username ?? '')}</p><p>${escapeHtml(formatForumDate(post.created_at))}</p>${post.cooked ?? ''}</div>`)
524
+ return parts.join('\n')
525
+ }
526
+ } catch {
527
+ return null
528
+ }
529
+ return null
530
+ }
531
+
532
+ function escapeHtml(text: string): string {
533
+ return text.replace(/[&<>"]/g, ch => ({ '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;' })[ch] ?? ch)
534
+ }
535
+
536
+ // ---------------------------------------------------------- generation ----
537
+
538
+ /** Everything candidate generation learned about the page, for features. */
539
+ export interface GenerationFacts {
540
+ readabilityNull: boolean
541
+ gatePass: boolean
542
+ pruneChanged: boolean
543
+ noscriptText: number
544
+ record: RecordRegion | null
545
+ hasMain: boolean
546
+ hasPreloaded: boolean
547
+ /** The semantic main container, when found. */
548
+ main: Element | null
549
+ }
550
+
551
+ /** Below this much noscript text the noscript-unwrapped paths are skipped. */
552
+ const NOSCRIPT_MIN_TEXT_CHARS = 200
553
+
554
+ /**
555
+ * Propose every candidate extraction for one parsed page.
556
+ * @param document - the parsed page (data URIs already elided); not mutated.
557
+ */
558
+ export function generateCandidates(document: Document): { candidates: RawCandidate[], facts: GenerationFacts } {
559
+ const candidates: RawCandidate[] = []
560
+ const body = document.body
561
+ const bodyHtml = body?.innerHTML ?? ''
562
+
563
+ // Rails are marked before Readability (it drops the class names that
564
+ // identify them) and cut from its output; the gate and `artu` see the
565
+ // extraction exactly as unmarked, so the legacy marker and the "keep
566
+ // everything" alternative are unchanged.
567
+ const marked = runReadability(markRecommendationRails(document) ?? document)
568
+ const rails = marked === null ? null : splitRails(marked.content, document)
569
+ const art = marked === null || rails === null ? null : { ...marked, content: rails.plain }
570
+ const gatePass = art !== null && articleCoversPage(art.content, document)
571
+ let pruneChanged = false
572
+ if (art !== null) {
573
+ const pruned = prunedArticleContent(rails?.derailed ?? art.content, document)
574
+ pruneChanged = pruned !== art.content
575
+ candidates.push({ kind: 'art', html: pruned, title: art.title, legacy: gatePass })
576
+ if (pruneChanged) candidates.push({ kind: 'artu', html: art.content, title: art.title, legacy: false })
577
+ }
578
+ // `linkDensityModifier` is a public Readability option missing from the
579
+ // bundled type declarations; it relaxes the link-density cleaning cutoffs.
580
+ const lenient = runReadability(document, { linkDensityModifier: 0.4 } as ConstructorParameters<typeof Readability>[1])
581
+ if (lenient !== null && lenient.content !== art?.content) {
582
+ candidates.push({ kind: 'artx', html: lenient.content, title: lenient.title, legacy: false })
583
+ }
584
+ const title = art?.title ?? lenient?.title
585
+
586
+ const main = body === null ? null : findMainContainer(document)
587
+ if (main !== null) candidates.push({ kind: 'main', html: main.innerHTML, title, legacy: false })
588
+
589
+ candidates.push({ kind: 'doc', html: bodyHtml, legacy: !gatePass })
590
+
591
+ if (body !== null) {
592
+ const cleaned = body.cloneNode(true) as Element
593
+ stripPageChrome(cleaned)
594
+ candidates.push({ kind: 'docc', html: cleaned.innerHTML, legacy: false })
595
+ }
596
+
597
+ const record = body === null ? null : findRecordRegion(body)
598
+ if (record !== null) {
599
+ const heading = document.querySelector('h1')
600
+ const html = (heading !== null ? heading.outerHTML : '') + record.items.map(item => item.outerHTML).join('\n')
601
+ candidates.push({ kind: 'rec', html, legacy: false })
602
+ }
603
+
604
+ const noscriptText = noscriptTextLength(document)
605
+ if (noscriptText >= NOSCRIPT_MIN_TEXT_CHARS) {
606
+ const unwrapped = document.cloneNode(true) as Document
607
+ unwrapNoscripts(unwrapped)
608
+ const nsArt = runReadability(unwrapped)
609
+ if (nsArt !== null) candidates.push({ kind: 'ns_art', html: prunedArticleContent(nsArt.content, unwrapped), title: nsArt.title, legacy: false })
610
+ candidates.push({ kind: 'ns_doc', html: unwrapped.body?.innerHTML ?? '', legacy: false })
611
+ }
612
+
613
+ const preloaded = discoursePreloadedHtml(document)
614
+ if (preloaded !== null) candidates.push({ kind: 'pre', html: preloaded, title, legacy: false })
615
+
616
+ // Identical proposals (a `<main>` that IS Readability's pick, a chrome
617
+ // strip that removed nothing) would only cost another sanitize pass.
618
+ const seen = new Set(candidates.filter(candidate => candidate.legacy).map(candidate => candidate.html))
619
+ const unique = candidates.filter(candidate => {
620
+ if (candidate.legacy) return true
621
+ if (seen.has(candidate.html)) return false
622
+ seen.add(candidate.html)
623
+ return true
624
+ })
625
+
626
+ return {
627
+ candidates: unique,
628
+ facts: {
629
+ readabilityNull: art === null,
630
+ gatePass,
631
+ pruneChanged,
632
+ noscriptText,
633
+ record,
634
+ hasMain: main !== null,
635
+ hasPreloaded: preloaded !== null,
636
+ main,
637
+ },
638
+ }
639
+ }
package/src/index.ts CHANGED
@@ -51,10 +51,12 @@ export type { ChallengeVerdict } from './challenge.ts'
51
51
  export { deepCaptureWalk } from './autoscroll.ts'
52
52
  export type { DeepCapture, DeepCaptureStats } from './autoscroll.ts'
53
53
  export { PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider, WEB_FETCH_CHALLENGE_CODE } from './provider.ts'
54
- export { htmlToMarkdown } from './markdown.ts'
54
+ export { blockTrainingRows, htmlToMarkdown, renderCandidate, scoreCandidates } from './markdown.ts'
55
+ export { BLOCK_FEATURES, setBlockModel } from './blocks.ts'
55
56
  export { extractPdfMarkdown, isPdfBytes, isPdfContentType, layoutPdfMarkdown, mayBePdf, MAX_PDF_BYTES, MAX_PDF_PAGES, readPdfDocument, WEB_PDF_ENCRYPTED_CODE, WEB_PDF_INVALID_CODE, WEB_PDF_NO_TEXT_CODE, WEB_PDF_TOO_LARGE_CODE } from './pdf.ts'
56
57
  export type { PdfDocumentText, PdfExtraction, PdfExtractOptions, PdfOutlineEntry, PdfPageText, PdfTextRun } from './pdf.ts'
57
- export type { DenoiseMode, DenoiseResult } from './markdown.ts'
58
+ export type { DenoiseMode, DenoiseResult, ScoredCandidate } from './markdown.ts'
59
+ export { FEATURE_NAMES, PAGE_FEATURES } from './selector.ts'
58
60
 
59
61
  /** Cordis plugin name used by loader diagnostics. */
60
62
  export const name = 'dsh-web-fetch-playwright'