dsh-web-fetch-playwright 0.2.16 → 0.2.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +47 -4
- package/README.zh-CN.md +47 -4
- package/lib/client.js +45 -20
- package/lib/index.js +1869 -75
- package/lib/types/autoscroll.d.ts +277 -0
- package/lib/types/client/controller.d.ts +7 -0
- package/lib/types/client/locales.d.ts +1 -1
- package/lib/types/config.d.ts +38 -3
- package/lib/types/index.d.ts +5 -1
- package/lib/types/markdown.d.ts +8 -5
- package/lib/types/pdf.d.ts +143 -0
- package/lib/types/provider.d.ts +34 -0
- package/lib/types/strip.d.ts +59 -0
- package/lib/types/types.d.ts +5 -0
- package/package.json +3 -2
- package/src/autoscroll.ts +653 -0
- package/src/client/card.tsx +13 -0
- package/src/client/controller.ts +9 -0
- package/src/client/locales.ts +7 -0
- package/src/config.ts +46 -3
- package/src/index.ts +7 -0
- package/src/markdown.ts +252 -6
- package/src/pdf.ts +1114 -0
- package/src/provider.ts +174 -76
- package/src/session-window.ts +8 -0
- package/src/strip.ts +72 -0
- package/src/types.ts +5 -0
package/src/client/controller.ts
CHANGED
|
@@ -46,6 +46,11 @@ export interface PlaywrightSettings {
|
|
|
46
46
|
* same tab (0–60000); 0 = off (return the first response as-is).
|
|
47
47
|
*/
|
|
48
48
|
challengeWaitMs?: number
|
|
49
|
+
/**
|
|
50
|
+
* Bounded budget (ms) for the deep-capture scroll walk (0–30000); 0 = off
|
|
51
|
+
* (capture the document as it rests, the legacy single-instant behavior).
|
|
52
|
+
*/
|
|
53
|
+
scrollBudgetMs?: number
|
|
49
54
|
}
|
|
50
55
|
|
|
51
56
|
/** What the Playwright card renders. */
|
|
@@ -65,6 +70,8 @@ export interface PlaywrightCardState extends CardShell {
|
|
|
65
70
|
maxConcurrency: CardFieldState
|
|
66
71
|
/** Challenge wait input (draft decimal integer of milliseconds). */
|
|
67
72
|
challengeWaitMs: CardFieldState
|
|
73
|
+
/** Scroll-walk budget input (draft decimal integer of milliseconds). */
|
|
74
|
+
scrollBudgetMs: CardFieldState
|
|
68
75
|
}
|
|
69
76
|
|
|
70
77
|
/** The registration-side face the card's slot entry injects. */
|
|
@@ -97,6 +104,7 @@ export class PlaywrightCardController {
|
|
|
97
104
|
checkboxField('denoise'),
|
|
98
105
|
numberField('maxConcurrency', 1, 200),
|
|
99
106
|
numberField('challengeWaitMs', 0, 60_000),
|
|
107
|
+
numberField('scrollBudgetMs', 0, 30_000),
|
|
100
108
|
],
|
|
101
109
|
)
|
|
102
110
|
this.store = this.form.bind(() => this.projection())
|
|
@@ -113,6 +121,7 @@ export class PlaywrightCardController {
|
|
|
113
121
|
denoise: this.form.field('denoise'),
|
|
114
122
|
maxConcurrency: this.form.field('maxConcurrency'),
|
|
115
123
|
challengeWaitMs: this.form.field('challengeWaitMs'),
|
|
124
|
+
scrollBudgetMs: this.form.field('scrollBudgetMs'),
|
|
116
125
|
}
|
|
117
126
|
}
|
|
118
127
|
|
package/src/client/locales.ts
CHANGED
|
@@ -18,6 +18,7 @@ export type PlaywrightCardLocaleKey =
|
|
|
18
18
|
| 'denoise' | 'denoiseHint'
|
|
19
19
|
| 'maxConcurrency' | 'maxConcurrencyHint' | 'maxConcurrencyPlaceholder'
|
|
20
20
|
| 'challengeWaitMs' | 'challengeWaitMsHint' | 'challengeWaitMsPlaceholder'
|
|
21
|
+
| 'scrollBudgetMs' | 'scrollBudgetMsHint' | 'scrollBudgetMsPlaceholder'
|
|
21
22
|
| 'overridden' | 'reset' | 'readOnly'
|
|
22
23
|
| 'save' | 'saving' | 'discard' | 'saveFailed' | 'invalidText'
|
|
23
24
|
|
|
@@ -59,6 +60,9 @@ export const en: Record<PlaywrightCardLocaleKey, string> = {
|
|
|
59
60
|
challengeWaitMs: 'Cloudflare challenge wait (ms)',
|
|
60
61
|
challengeWaitMsHint: 'Bounded wait for a Cloudflare challenge to clear naturally in the same tab (0–60000; 0 = off). 0 disables: the first response is returned as-is.',
|
|
61
62
|
challengeWaitMsPlaceholder: '(default: 15000)',
|
|
63
|
+
scrollBudgetMs: 'Deep-capture scroll budget (ms)',
|
|
64
|
+
scrollBudgetMsHint: 'Bounded in-page scroll walk that captures viewport-virtualized feeds (forum topics, comment threads) whole: posts outside the render window are empty placeholders at rest. 0–30000; 0 = off (capture the page as it rests).',
|
|
65
|
+
scrollBudgetMsPlaceholder: '(default: 20000)',
|
|
62
66
|
overridden: 'Overridden',
|
|
63
67
|
reset: 'Reset to default',
|
|
64
68
|
readOnly: 'This deployment stores settings read-only.',
|
|
@@ -100,6 +104,9 @@ export const zh: Record<PlaywrightCardLocaleKey, string> = {
|
|
|
100
104
|
challengeWaitMs: 'Cloudflare 挑战等待上限(毫秒)',
|
|
101
105
|
challengeWaitMsHint: '在同一标签页内有界等待 Cloudflare 验证自然通过(0–60000;0 = 关闭)。关闭时直接返回首次响应——旧版行为。',
|
|
102
106
|
challengeWaitMsPlaceholder: '(默认:15000)',
|
|
107
|
+
scrollBudgetMs: '深度抓取滚动预算(毫秒)',
|
|
108
|
+
scrollBudgetMsHint: '有界的页内滚动遍历,完整捕获视口虚拟化的信息流(论坛帖子、评论区):静止时渲染窗口外的帖子只是空占位符。0–30000;0 = 关闭(按页面静止状态抓取)。',
|
|
109
|
+
scrollBudgetMsPlaceholder: '(默认:20000)',
|
|
103
110
|
overridden: '已覆盖',
|
|
104
111
|
reset: '恢复默认',
|
|
105
112
|
readOnly: '本部署的设置为只读。',
|
package/src/config.ts
CHANGED
|
@@ -16,6 +16,19 @@
|
|
|
16
16
|
import type { Volatile } from '@deepseek-ai/cordis'
|
|
17
17
|
import z from '@deepseek-ai/schemastery'
|
|
18
18
|
|
|
19
|
+
/**
|
|
20
|
+
* Default bounded budget (ms) for the deep-capture scroll walk
|
|
21
|
+
* (`src/autoscroll.ts`). Sized so long topics stay fully covered at the
|
|
22
|
+
* careful cadence (300ms motion + 300ms rest per ~0.7-viewport step, plus
|
|
23
|
+
* the bottom patience window): 20s walks ~25kpx with pagination, covering
|
|
24
|
+
* the measured linux.do badcase (~18kpx, 51 replies) with headroom.
|
|
25
|
+
* Scroll-inert pages take the walk's fast path and spend a few seconds.
|
|
26
|
+
*/
|
|
27
|
+
export const DEFAULT_SCROLL_BUDGET_MS = 20_000
|
|
28
|
+
|
|
29
|
+
/** Ceiling the schema accepts for `scrollBudgetMs`. */
|
|
30
|
+
export const MAX_SCROLL_BUDGET_MS = 30_000
|
|
31
|
+
|
|
19
32
|
/** Default CDP endpoint when the settings section leaves it blank. */
|
|
20
33
|
export const DEFAULT_CDP_ENDPOINT = '127.0.0.1:9222'
|
|
21
34
|
|
|
@@ -109,6 +122,15 @@ export interface Config {
|
|
|
109
122
|
* Schema default: {@link DEFAULT_CHALLENGE_RETRIES}.
|
|
110
123
|
*/
|
|
111
124
|
challengeRetries?: number
|
|
125
|
+
/**
|
|
126
|
+
* Bounded budget (ms) for the deep-capture scroll walk — the in-page pass
|
|
127
|
+
* that defeats viewport virtualization on feed-shaped pages (Discourse
|
|
128
|
+
* topic streams: replies outside the render window are empty height-only
|
|
129
|
+
* placeholders carrying the post's id; see `src/autoscroll.ts`). `0`
|
|
130
|
+
* restores the legacy single-instant capture.
|
|
131
|
+
* Schema default: {@link DEFAULT_SCROLL_BUDGET_MS}.
|
|
132
|
+
*/
|
|
133
|
+
scrollBudgetMs?: number
|
|
112
134
|
/**
|
|
113
135
|
* How many fetches may render at once. Blank = backend default (4 for
|
|
114
136
|
* local — each slot launches a browser; 50 for CDP — each slot is a tab in
|
|
@@ -135,6 +157,8 @@ export const Config = z.object({
|
|
|
135
157
|
// The bounded natural-wait knobs; 0 disables the whole challenge path.
|
|
136
158
|
challengeWaitMs: z.number().step(100).min(0).max(MAX_CHALLENGE_WAIT_MS).default(DEFAULT_CHALLENGE_WAIT_MS).volatile(),
|
|
137
159
|
challengeRetries: z.number().step(1).min(0).max(MAX_CHALLENGE_RETRIES).default(DEFAULT_CHALLENGE_RETRIES).volatile(),
|
|
160
|
+
// The deep-capture scroll walk's bounded budget; 0 disables the walk.
|
|
161
|
+
scrollBudgetMs: z.number().step(100).min(0).max(MAX_SCROLL_BUDGET_MS).default(DEFAULT_SCROLL_BUDGET_MS).volatile(),
|
|
138
162
|
})
|
|
139
163
|
|
|
140
164
|
/**
|
|
@@ -164,6 +188,8 @@ export interface PlaywrightPluginConfig {
|
|
|
164
188
|
challengeWaitMs: Volatile<number>
|
|
165
189
|
/** Challenge retry count as a live reference. */
|
|
166
190
|
challengeRetries: Volatile<number>
|
|
191
|
+
/** Deep-capture scroll-walk budget (ms) as a live reference. */
|
|
192
|
+
scrollBudgetMs: Volatile<number>
|
|
167
193
|
}
|
|
168
194
|
|
|
169
195
|
/**
|
|
@@ -185,6 +211,7 @@ export function snapshotsOf(config: PlaywrightPluginConfig): Config {
|
|
|
185
211
|
maxConcurrency: config.maxConcurrency.get(),
|
|
186
212
|
challengeWaitMs: config.challengeWaitMs.get(),
|
|
187
213
|
challengeRetries: config.challengeRetries.get(),
|
|
214
|
+
scrollBudgetMs: config.scrollBudgetMs.get(),
|
|
188
215
|
}
|
|
189
216
|
}
|
|
190
217
|
|
|
@@ -210,15 +237,19 @@ export function resolveConfig(config: Partial<Config>): ResolvedConfig {
|
|
|
210
237
|
maxConcurrency: resolved.maxConcurrency.get(),
|
|
211
238
|
challengeWaitMs: resolved.challengeWaitMs.get(),
|
|
212
239
|
challengeRetries: resolved.challengeRetries.get(),
|
|
240
|
+
scrollBudgetMs: resolved.scrollBudgetMs.get(),
|
|
213
241
|
}
|
|
214
242
|
}
|
|
215
243
|
|
|
216
244
|
/**
|
|
217
245
|
* Complete config after schemastery applies the field defaults it owns.
|
|
218
|
-
* `maxConcurrency`
|
|
219
|
-
*
|
|
246
|
+
* `maxConcurrency` and `scrollBudgetMs` stay optional — the former because
|
|
247
|
+
* its effective default depends on `backend`, the latter so hand-built
|
|
248
|
+
* resolved configs (tests) may omit it and let
|
|
249
|
+
* {@link effectiveScrollBudgetMs} supply the default the schema would have.
|
|
220
250
|
*/
|
|
221
|
-
export type ResolvedConfig = Omit<Required<Config>, 'maxConcurrency'
|
|
251
|
+
export type ResolvedConfig = Omit<Required<Config>, 'maxConcurrency' | 'scrollBudgetMs'>
|
|
252
|
+
& { maxConcurrency?: number; scrollBudgetMs?: number }
|
|
222
253
|
|
|
223
254
|
/**
|
|
224
255
|
* The concurrency limit a fetch actually runs with: an explicit setting
|
|
@@ -257,6 +288,18 @@ export function effectiveChallengeRetries(config: Pick<Config, 'challengeRetries
|
|
|
257
288
|
return DEFAULT_CHALLENGE_RETRIES
|
|
258
289
|
}
|
|
259
290
|
|
|
291
|
+
/**
|
|
292
|
+
* The scroll-walk budget a fetch actually runs with: an explicit setting
|
|
293
|
+
* wins, else the schema default. `0` means the deep-capture walk is off.
|
|
294
|
+
*
|
|
295
|
+
* @param config - the resolved settings section (or any partial of it).
|
|
296
|
+
* @returns the bounded walk budget in milliseconds.
|
|
297
|
+
*/
|
|
298
|
+
export function effectiveScrollBudgetMs(config: Pick<Config, 'scrollBudgetMs'>): number {
|
|
299
|
+
if (typeof config.scrollBudgetMs === 'number') return config.scrollBudgetMs
|
|
300
|
+
return DEFAULT_SCROLL_BUDGET_MS
|
|
301
|
+
}
|
|
302
|
+
|
|
260
303
|
/**
|
|
261
304
|
* The context mode a fetch actually runs with: profile (a tab in the remote
|
|
262
305
|
* browser's real profile) requires the CDP backend — a local launch has no
|
package/src/index.ts
CHANGED
|
@@ -26,9 +26,12 @@ export {
|
|
|
26
26
|
DEFAULT_CDP_ENDPOINT,
|
|
27
27
|
DEFAULT_CHALLENGE_RETRIES,
|
|
28
28
|
DEFAULT_CHALLENGE_WAIT_MS,
|
|
29
|
+
DEFAULT_SCROLL_BUDGET_MS,
|
|
30
|
+
MAX_SCROLL_BUDGET_MS,
|
|
29
31
|
effectiveChallengeRetries,
|
|
30
32
|
effectiveChallengeWaitMs,
|
|
31
33
|
effectiveContextMode,
|
|
34
|
+
effectiveScrollBudgetMs,
|
|
32
35
|
normalizeCdpEndpoint,
|
|
33
36
|
resolveConfig,
|
|
34
37
|
snapshotsOf,
|
|
@@ -45,8 +48,12 @@ export { CdpConnectionPool } from './cdp-pool.ts'
|
|
|
45
48
|
export type { CdpAcquireMode, CdpConnect, CdpLease } from './cdp-pool.ts'
|
|
46
49
|
export { CHALLENGE_DOM_PROBE, CHALLENGE_FINISH_RESERVE_MS, CHALLENGE_POLL_INTERVAL_MS, CHALLENGE_TITLE_RE, classifyChallengeHtml, classifyChallengeResponse, isChallengeCompatibleResponse } from './challenge.ts'
|
|
47
50
|
export type { ChallengeVerdict } from './challenge.ts'
|
|
51
|
+
export { deepCaptureWalk } from './autoscroll.ts'
|
|
52
|
+
export type { DeepCapture, DeepCaptureStats } from './autoscroll.ts'
|
|
48
53
|
export { PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider, WEB_FETCH_CHALLENGE_CODE } from './provider.ts'
|
|
49
54
|
export { htmlToMarkdown } from './markdown.ts'
|
|
55
|
+
export { extractPdfMarkdown, isPdfBytes, isPdfContentType, layoutPdfMarkdown, mayBePdf, MAX_PDF_BYTES, MAX_PDF_PAGES, readPdfDocument, WEB_PDF_ENCRYPTED_CODE, WEB_PDF_INVALID_CODE, WEB_PDF_NO_TEXT_CODE, WEB_PDF_TOO_LARGE_CODE } from './pdf.ts'
|
|
56
|
+
export type { PdfDocumentText, PdfExtraction, PdfExtractOptions, PdfOutlineEntry, PdfPageText, PdfTextRun } from './pdf.ts'
|
|
50
57
|
export type { DenoiseMode, DenoiseResult } from './markdown.ts'
|
|
51
58
|
|
|
52
59
|
/** Cordis plugin name used by loader diagnostics. */
|
package/src/markdown.ts
CHANGED
|
@@ -8,11 +8,14 @@
|
|
|
8
8
|
* footers, and ad chrome by scoring link density and text mass) — its result
|
|
9
9
|
* must then pass a coverage gate that distrusts tiny or sliver-thin
|
|
10
10
|
* extractions, which on list-shaped pages (portals, catalogs) is exactly
|
|
11
|
-
* what the link-density penalty leaves behind;
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
11
|
+
* what the link-density penalty leaves behind; an accepted extraction is
|
|
12
|
+
* then pruned of trailing link-dense sections — the post-article
|
|
13
|
+
* recommendation rails and share bars the scorer's vocabularies cannot name
|
|
14
|
+
* (they arrive as bare divs with no class to match); DOMPurify sanitizes
|
|
15
|
+
* whatever HTML remains and forbids the layout tags noise lives in;
|
|
16
|
+
* Turndown with the GFM plugin converts to markdown using the same style
|
|
17
|
+
* options and span-safe table rules as the shipped `dsh-tool-web` renderer,
|
|
18
|
+
* so output is consistent with what `web_fetch` produces elsewhere.
|
|
16
19
|
*
|
|
17
20
|
* Synchronous for its callers and unit-tested against fixture pages: jsdom
|
|
18
21
|
* itself is loaded lazily on the first call — and, inside a dsh profile, only
|
|
@@ -280,6 +283,245 @@ export interface DenoiseResult {
|
|
|
280
283
|
mode: DenoiseMode
|
|
281
284
|
}
|
|
282
285
|
|
|
286
|
+
/**
|
|
287
|
+
* Count non-whitespace characters under `root` that sit inside an anchor,
|
|
288
|
+
* skipping the same noise subtrees {@link visibleTextLength} skips so the
|
|
289
|
+
* numerator and denominator of a link-density ratio stay comparable.
|
|
290
|
+
*
|
|
291
|
+
* @param root - the subtree to measure; `null` counts as nothing.
|
|
292
|
+
* @returns the number of non-whitespace characters inside `<a>` descendants.
|
|
293
|
+
*/
|
|
294
|
+
function linkTextLength(root: Node | null): number {
|
|
295
|
+
if (root === null) return 0
|
|
296
|
+
let length = 0
|
|
297
|
+
const visit = (node: Node, insideAnchor: boolean): void => {
|
|
298
|
+
if (node.nodeType === 3) {
|
|
299
|
+
if (insideAnchor) length += (node.nodeValue ?? '').replace(/\s+/g, '').length
|
|
300
|
+
return
|
|
301
|
+
}
|
|
302
|
+
if (node.nodeType !== 1 || NOISE_SUBTREE_TAGS.has(node.nodeName.toLowerCase())) return
|
|
303
|
+
const anchor = insideAnchor || node.nodeName.toLowerCase() === 'a'
|
|
304
|
+
for (let child = node.firstChild; child !== null; child = child.nextSibling) visit(child, anchor)
|
|
305
|
+
}
|
|
306
|
+
visit(root, false)
|
|
307
|
+
return length
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
/**
|
|
311
|
+
* Minimum non-whitespace characters a block must carry before its link
|
|
312
|
+
* density is even evaluated. Blocks below this are spacers, empty wrappers,
|
|
313
|
+
* or one-line sign-offs — too small to be worth a removal decision, so the
|
|
314
|
+
* trailing walk skips past them instead of stopping at them.
|
|
315
|
+
*/
|
|
316
|
+
const TRAILING_MIN_TEXT_CHARS = 40
|
|
317
|
+
|
|
318
|
+
/**
|
|
319
|
+
* Link-density share above which a trailing block reads as a link farm:
|
|
320
|
+
* recommendation feeds, related-article rails, tag clouds, share bars.
|
|
321
|
+
* Measured on the 163.com dy-article badcase the section's pick carried 84
|
|
322
|
+
* linked characters inside 183 total (0.46) — recommendation cards dilute
|
|
323
|
+
* their density with author/date lines, so a 0.5 bar would miss them by
|
|
324
|
+
* construction; an honest closing paragraph with a citation or two measures
|
|
325
|
+
* 0.1–0.3, leaving 0.4 in the empty band between.
|
|
326
|
+
*/
|
|
327
|
+
const TRAILING_LINK_DENSITY = 0.4
|
|
328
|
+
|
|
329
|
+
/**
|
|
330
|
+
* A trailing block qualifies as a link farm only when it is link-heavy in
|
|
331
|
+
* aggregate too: at least {@link TRAILING_LINK_COUNT} anchors or
|
|
332
|
+
* {@link TRAILING_LINK_TEXT_CHARS} linked characters. Density alone would
|
|
333
|
+
* let a single long linked sentence (an inline citation) drag a legitimate
|
|
334
|
+
* paragraph under the knife.
|
|
335
|
+
*/
|
|
336
|
+
const TRAILING_LINK_COUNT = 3
|
|
337
|
+
const TRAILING_LINK_TEXT_CHARS = 60
|
|
338
|
+
|
|
339
|
+
/**
|
|
340
|
+
* A trailing block may never hold more than this share of the whole
|
|
341
|
+
* article's text. The prune targets *appended* widgets; a block that IS most
|
|
342
|
+
* of the extraction says the scorer picked the widget, which is the coverage
|
|
343
|
+
* gate's failure mode to handle, not this walk's.
|
|
344
|
+
*/
|
|
345
|
+
const TRAILING_MAX_ARTICLE_SHARE = 0.6
|
|
346
|
+
|
|
347
|
+
/**
|
|
348
|
+
* A trailing run may never hold more than this share of its container's
|
|
349
|
+
* text — the aggregate form of the guard above, and the one that keeps
|
|
350
|
+
* list-shaped extractions whole. A recommendation rail is a *minority*
|
|
351
|
+
* appended to real content (the 163 rail is 44% of its container); a
|
|
352
|
+
* portal or feed whose extraction IS a list of link rows (measured on a
|
|
353
|
+
* Hacker News homepage capture: ~91 trailing rows, every one individually
|
|
354
|
+
* link-dense at 0.69–0.94) has a tail run holding ~97% of the container —
|
|
355
|
+
* there the tail IS the content, the walk declines wholesale, and only a
|
|
356
|
+
* run that is a minority of its container is removed.
|
|
357
|
+
*/
|
|
358
|
+
const TRAILING_MAX_RUN_SHARE = 0.6
|
|
359
|
+
|
|
360
|
+
/**
|
|
361
|
+
* Link-density band within which a run and the block the walk stopped at
|
|
362
|
+
* read as the same material. When the content immediately before the run
|
|
363
|
+
* is itself as link-dense as the run — measured on a Hacker News homepage
|
|
364
|
+
* capture, the rows after the run's cut measure 0.94 against the run's
|
|
365
|
+
* 0.91 — the container is a *uniform list* (feed rows, portal headlines)
|
|
366
|
+
* and the tail is that list's own content, not a widget appended after it;
|
|
367
|
+
* a widget hangs after genuinely different material (the 163 rail's 0.44
|
|
368
|
+
* follows body text at 0.00). Only a run denser than what precedes it is
|
|
369
|
+
* removed.
|
|
370
|
+
*/
|
|
371
|
+
const TRAILING_UNIFORM_DENSITY_BAND = 0.25
|
|
372
|
+
|
|
373
|
+
/**
|
|
374
|
+
* Prune trailing link-dense sections from an accepted Readability
|
|
375
|
+
* extraction — the post-article widget class the scorer itself cannot be
|
|
376
|
+
* taught to drop.
|
|
377
|
+
*
|
|
378
|
+
* Why this exists (badcase: `https://www.163.com/dy/article/…`): Chinese
|
|
379
|
+
* portal article pages append a recommendation feed inside the very wrapper
|
|
380
|
+
* Readability scores as the article (`#content` holds the body, the
|
|
381
|
+
* platform disclaimer, then a rail of image/title/author/date cards). Every
|
|
382
|
+
* existing noise layer misses it: Readability's `unlikelyCandidates` /
|
|
383
|
+
* `negative` vocabularies know `related` but not `recommend` (nor the
|
|
384
|
+
* pinyin families `tuijian`/`xiangguan`), and its own cleaning pass draws
|
|
385
|
+
* the "mostly links" line at per-block density 0.5 — these cards measure
|
|
386
|
+
* 0.42–0.48 (the title links over title+author+date text), so every card
|
|
387
|
+
* clears it; in the extracted HTML the rail is a *bare* div — class and id
|
|
388
|
+
* are gone — so no class-based rule can ever see it again; DOMPurify's
|
|
389
|
+
* FORBID_TAGS works on tags, and the rail is plain div/p/h3/a; and the
|
|
390
|
+
* coverage gate cannot help because the noise rides *inside* the
|
|
391
|
+
* extraction it measures (the badcase passed at 0.103 — the rail's text
|
|
392
|
+
* was in both numerator and denominator). Modern sites with CSS-module
|
|
393
|
+
* class names (`css-1b3x7r`) present the same shape with no vocabulary to
|
|
394
|
+
* match at all, so the only signal that survives all of this is
|
|
395
|
+
* structural: a *trailing* run of blocks whose text is mostly links.
|
|
396
|
+
*
|
|
397
|
+
* For every container in the extraction the walk gathers the maximal
|
|
398
|
+
* trailing run of qualifying children — link-farm blocks per the constants
|
|
399
|
+
* above, plus sub-threshold spacers (empty wrapper divs) — and removes the
|
|
400
|
+
* run only when BOTH structural guards say "appended widget, not content":
|
|
401
|
+
* the run is a minority of the container's text
|
|
402
|
+
* ({@link TRAILING_MAX_RUN_SHARE}), and the run is clearly denser than the
|
|
403
|
+
* block the walk stopped at ({@link TRAILING_UNIFORM_DENSITY_BAND}). The
|
|
404
|
+
* guards split the two worlds the per-block view cannot: a rail is
|
|
405
|
+
* one-or-few appended blocks after genuinely different material (the 163
|
|
406
|
+
* rail's 0.44 density follows body text at 0.00), while a link-list
|
|
407
|
+
* extraction — feed rows, portal headlines — is *uniform*: its rows before
|
|
408
|
+
* any cut measure the same density as the rows after it (0.91 vs 0.94 on
|
|
409
|
+
* the Hacker News capture), so no tail of it can be mistaken for a widget.
|
|
410
|
+
*
|
|
411
|
+
* Deliberately NOT applied to the whole-document fallback: that path
|
|
412
|
+
* exists for list-shaped pages where the links *are* the content (the
|
|
413
|
+
* 163.com homepage fix), and pruning link-dense regions there would
|
|
414
|
+
* re-introduce exactly the recall loss the coverage gate exists to
|
|
415
|
+
* prevent.
|
|
416
|
+
*
|
|
417
|
+
* @param container - parsed `article.content`; mutated in place.
|
|
418
|
+
*/
|
|
419
|
+
function pruneTrailingLinkDenseSections(container: Element): void {
|
|
420
|
+
const articleText = visibleTextLength(container)
|
|
421
|
+
if (articleText === 0) return
|
|
422
|
+
// The container itself leads the walk — Readability usually wraps its
|
|
423
|
+
// content in div#readability-page-1, but an unwrapped extraction's
|
|
424
|
+
// top-level blocks are the container's direct children and must be
|
|
425
|
+
// examined too. querySelectorAll snapshots before any removal; entries
|
|
426
|
+
// inside an already-removed subtree are skipped by the contains() check,
|
|
427
|
+
// and parents are visited before children, so a deeper run inside a block
|
|
428
|
+
// an outer run is taking never competes with it.
|
|
429
|
+
for (const element of [container, ...Array.from(container.querySelectorAll('*'))]) {
|
|
430
|
+
if (!container.contains(element) && element !== container) continue
|
|
431
|
+
removeTrailingLinkDenseRun(element, articleText)
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
/**
|
|
436
|
+
* Remove one container's maximal trailing run of link-dense children when
|
|
437
|
+
* that run is a minority of the container's text AND clearly different
|
|
438
|
+
* material from the content that precedes it.
|
|
439
|
+
*
|
|
440
|
+
* @param parent - the container whose element children are examined from
|
|
441
|
+
* the end.
|
|
442
|
+
* @param articleText - the whole article's text, for the per-block share
|
|
443
|
+
* guard.
|
|
444
|
+
*/
|
|
445
|
+
function removeTrailingLinkDenseRun(parent: Element, articleText: number): void {
|
|
446
|
+
const children = Array.from(parent.children)
|
|
447
|
+
if (children.length === 0) return
|
|
448
|
+
const containerText = visibleTextLength(parent)
|
|
449
|
+
if (containerText === 0) return
|
|
450
|
+
const run: Element[] = []
|
|
451
|
+
let runText = 0
|
|
452
|
+
let runLinked = 0
|
|
453
|
+
let stopIndex = -1
|
|
454
|
+
for (let index = children.length - 1; index >= 0; index--) {
|
|
455
|
+
const child = children[index]
|
|
456
|
+
if (child === undefined) break
|
|
457
|
+
const text = visibleTextLength(child)
|
|
458
|
+
// Too small to judge (spacer, empty wrapper): it belongs to the run
|
|
459
|
+
// but cannot extend it — keep walking toward earlier siblings.
|
|
460
|
+
if (text < TRAILING_MIN_TEXT_CHARS) {
|
|
461
|
+
run.push(child)
|
|
462
|
+
continue
|
|
463
|
+
}
|
|
464
|
+
const linked = linkTextLength(child)
|
|
465
|
+
const linkCount = child.querySelectorAll('a').length
|
|
466
|
+
if (
|
|
467
|
+
linked / text >= TRAILING_LINK_DENSITY
|
|
468
|
+
&& (linkCount >= TRAILING_LINK_COUNT || linked >= TRAILING_LINK_TEXT_CHARS)
|
|
469
|
+
&& text <= TRAILING_MAX_ARTICLE_SHARE * articleText
|
|
470
|
+
) {
|
|
471
|
+
run.push(child)
|
|
472
|
+
runText += text
|
|
473
|
+
runLinked += linked
|
|
474
|
+
continue
|
|
475
|
+
}
|
|
476
|
+
// Real content that is not a link farm: everything before it is
|
|
477
|
+
// anchored by actual article flow — stop and compare against it.
|
|
478
|
+
stopIndex = index
|
|
479
|
+
break
|
|
480
|
+
}
|
|
481
|
+
if (runText === 0 || stopIndex === -1) {
|
|
482
|
+
// Nothing to remove — or the run reached the top of the container (no
|
|
483
|
+
// preceding content to differ from): a tail that IS the whole
|
|
484
|
+
// container is list content, never a widget.
|
|
485
|
+
return
|
|
486
|
+
}
|
|
487
|
+
// The tail is the same material as what precedes it (a uniform list's
|
|
488
|
+
// rows): decline wholesale.
|
|
489
|
+
const stopChild = children[stopIndex]
|
|
490
|
+
if (stopChild === undefined) return
|
|
491
|
+
const stopDensity = linkTextLength(stopChild) / Math.max(1, visibleTextLength(stopChild))
|
|
492
|
+
if (Math.abs(runLinked / runText - stopDensity) <= TRAILING_UNIFORM_DENSITY_BAND) return
|
|
493
|
+
// The tail holds most of the container (a link-list page's rows): decline
|
|
494
|
+
// wholesale.
|
|
495
|
+
if (runText > TRAILING_MAX_RUN_SHARE * containerText) return
|
|
496
|
+
for (const child of run) child.remove()
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
/**
|
|
500
|
+
* Run {@link pruneTrailingLinkDenseSections} over one extraction with a
|
|
501
|
+
* misfire guard: a prune that takes a healthy article below the gate's
|
|
502
|
+
* absolute floor ({@link GATE_MIN_ARTICLE_TEXT_CHARS}) or empties it
|
|
503
|
+
* entirely says the walk misread the page, and Readability's original pick
|
|
504
|
+
* is returned untouched. A *sparse* article (under the floor before the
|
|
505
|
+
* prune, auto-trusted by the gate) keeps its pruned form — its trailing
|
|
506
|
+
* links were noise all the same.
|
|
507
|
+
*
|
|
508
|
+
* @param articleHtml - the `article.content` HTML Readability produced.
|
|
509
|
+
* @param document - the live document, for parsing the fragment.
|
|
510
|
+
* @returns the pruned article HTML, or the original on a misfire.
|
|
511
|
+
*/
|
|
512
|
+
function prunedArticleContent(articleHtml: string, document: Document): string {
|
|
513
|
+
const container = document.createElement('div')
|
|
514
|
+
container.innerHTML = articleHtml
|
|
515
|
+
const before = visibleTextLength(container)
|
|
516
|
+
if (before === 0) return articleHtml
|
|
517
|
+
pruneTrailingLinkDenseSections(container)
|
|
518
|
+
const after = visibleTextLength(container)
|
|
519
|
+
if (after === 0 || (before >= GATE_MIN_ARTICLE_TEXT_CHARS && after < GATE_MIN_ARTICLE_TEXT_CHARS)) {
|
|
520
|
+
return articleHtml
|
|
521
|
+
}
|
|
522
|
+
return container.innerHTML
|
|
523
|
+
}
|
|
524
|
+
|
|
283
525
|
/**
|
|
284
526
|
* Convert one rendered HTML document to denoised markdown.
|
|
285
527
|
*
|
|
@@ -323,7 +565,11 @@ export function htmlToMarkdown(html: string, url: string): DenoiseResult {
|
|
|
323
565
|
article !== null && typeof article.content === 'string' && article.content !== ''
|
|
324
566
|
&& articleCoversPage(article.content, document)
|
|
325
567
|
) {
|
|
326
|
-
|
|
568
|
+
// Gate first, prune second, and in that order on purpose: the gate
|
|
569
|
+
// must judge Readability's pick as delivered (the widget rides inside
|
|
570
|
+
// it), and the prune then strips the post-article link farms the
|
|
571
|
+
// scorer could not be taught — see pruneTrailingLinkDenseSections.
|
|
572
|
+
source = prunedArticleContent(article.content, document)
|
|
327
573
|
title = article.title ?? undefined
|
|
328
574
|
}
|
|
329
575
|
} catch {
|