dsh-web-fetch-playwright 0.2.16 → 0.2.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -46,6 +46,11 @@ export interface PlaywrightSettings {
46
46
  * same tab (0–60000); 0 = off (return the first response as-is).
47
47
  */
48
48
  challengeWaitMs?: number
49
+ /**
50
+ * Bounded budget (ms) for the deep-capture scroll walk (0–30000); 0 = off
51
+ * (capture the document as it rests, the legacy single-instant behavior).
52
+ */
53
+ scrollBudgetMs?: number
49
54
  }
50
55
 
51
56
  /** What the Playwright card renders. */
@@ -65,6 +70,8 @@ export interface PlaywrightCardState extends CardShell {
65
70
  maxConcurrency: CardFieldState
66
71
  /** Challenge wait input (draft decimal integer of milliseconds). */
67
72
  challengeWaitMs: CardFieldState
73
+ /** Scroll-walk budget input (draft decimal integer of milliseconds). */
74
+ scrollBudgetMs: CardFieldState
68
75
  }
69
76
 
70
77
  /** The registration-side face the card's slot entry injects. */
@@ -97,6 +104,7 @@ export class PlaywrightCardController {
97
104
  checkboxField('denoise'),
98
105
  numberField('maxConcurrency', 1, 200),
99
106
  numberField('challengeWaitMs', 0, 60_000),
107
+ numberField('scrollBudgetMs', 0, 30_000),
100
108
  ],
101
109
  )
102
110
  this.store = this.form.bind(() => this.projection())
@@ -113,6 +121,7 @@ export class PlaywrightCardController {
113
121
  denoise: this.form.field('denoise'),
114
122
  maxConcurrency: this.form.field('maxConcurrency'),
115
123
  challengeWaitMs: this.form.field('challengeWaitMs'),
124
+ scrollBudgetMs: this.form.field('scrollBudgetMs'),
116
125
  }
117
126
  }
118
127
 
@@ -18,6 +18,7 @@ export type PlaywrightCardLocaleKey =
18
18
  | 'denoise' | 'denoiseHint'
19
19
  | 'maxConcurrency' | 'maxConcurrencyHint' | 'maxConcurrencyPlaceholder'
20
20
  | 'challengeWaitMs' | 'challengeWaitMsHint' | 'challengeWaitMsPlaceholder'
21
+ | 'scrollBudgetMs' | 'scrollBudgetMsHint' | 'scrollBudgetMsPlaceholder'
21
22
  | 'overridden' | 'reset' | 'readOnly'
22
23
  | 'save' | 'saving' | 'discard' | 'saveFailed' | 'invalidText'
23
24
 
@@ -59,6 +60,9 @@ export const en: Record<PlaywrightCardLocaleKey, string> = {
59
60
  challengeWaitMs: 'Cloudflare challenge wait (ms)',
60
61
  challengeWaitMsHint: 'Bounded wait for a Cloudflare challenge to clear naturally in the same tab (0–60000; 0 = off). 0 disables: the first response is returned as-is.',
61
62
  challengeWaitMsPlaceholder: '(default: 15000)',
63
+ scrollBudgetMs: 'Deep-capture scroll budget (ms)',
64
+ scrollBudgetMsHint: 'Bounded in-page scroll walk that captures viewport-virtualized feeds (forum topics, comment threads) whole: posts outside the render window are empty placeholders at rest. 0–30000; 0 = off (capture the page as it rests).',
65
+ scrollBudgetMsPlaceholder: '(default: 20000)',
62
66
  overridden: 'Overridden',
63
67
  reset: 'Reset to default',
64
68
  readOnly: 'This deployment stores settings read-only.',
@@ -100,6 +104,9 @@ export const zh: Record<PlaywrightCardLocaleKey, string> = {
100
104
  challengeWaitMs: 'Cloudflare 挑战等待上限(毫秒)',
101
105
  challengeWaitMsHint: '在同一标签页内有界等待 Cloudflare 验证自然通过(0–60000;0 = 关闭)。关闭时直接返回首次响应——旧版行为。',
102
106
  challengeWaitMsPlaceholder: '(默认:15000)',
107
+ scrollBudgetMs: '深度抓取滚动预算(毫秒)',
108
+ scrollBudgetMsHint: '有界的页内滚动遍历,完整捕获视口虚拟化的信息流(论坛帖子、评论区):静止时渲染窗口外的帖子只是空占位符。0–30000;0 = 关闭(按页面静止状态抓取)。',
109
+ scrollBudgetMsPlaceholder: '(默认:20000)',
103
110
  overridden: '已覆盖',
104
111
  reset: '恢复默认',
105
112
  readOnly: '本部署的设置为只读。',
package/src/config.ts CHANGED
@@ -16,6 +16,19 @@
16
16
  import type { Volatile } from '@deepseek-ai/cordis'
17
17
  import z from '@deepseek-ai/schemastery'
18
18
 
19
+ /**
20
+ * Default bounded budget (ms) for the deep-capture scroll walk
21
+ * (`src/autoscroll.ts`). Sized so long topics stay fully covered at the
22
+ * careful cadence (300ms motion + 300ms rest per ~0.7-viewport step, plus
23
+ * the bottom patience window): 20s walks ~25kpx with pagination, covering
24
+ * the measured linux.do badcase (~18kpx, 51 replies) with headroom.
25
+ * Scroll-inert pages take the walk's fast path and spend a few seconds.
26
+ */
27
+ export const DEFAULT_SCROLL_BUDGET_MS = 20_000
28
+
29
+ /** Ceiling the schema accepts for `scrollBudgetMs`. */
30
+ export const MAX_SCROLL_BUDGET_MS = 30_000
31
+
19
32
  /** Default CDP endpoint when the settings section leaves it blank. */
20
33
  export const DEFAULT_CDP_ENDPOINT = '127.0.0.1:9222'
21
34
 
@@ -109,6 +122,15 @@ export interface Config {
109
122
  * Schema default: {@link DEFAULT_CHALLENGE_RETRIES}.
110
123
  */
111
124
  challengeRetries?: number
125
+ /**
126
+ * Bounded budget (ms) for the deep-capture scroll walk — the in-page pass
127
+ * that defeats viewport virtualization on feed-shaped pages (Discourse
128
+ * topic streams: replies outside the render window are empty height-only
129
+ * placeholders carrying the post's id; see `src/autoscroll.ts`). `0`
130
+ * restores the legacy single-instant capture.
131
+ * Schema default: {@link DEFAULT_SCROLL_BUDGET_MS}.
132
+ */
133
+ scrollBudgetMs?: number
112
134
  /**
113
135
  * How many fetches may render at once. Blank = backend default (4 for
114
136
  * local — each slot launches a browser; 50 for CDP — each slot is a tab in
@@ -135,6 +157,8 @@ export const Config = z.object({
135
157
  // The bounded natural-wait knobs; 0 disables the whole challenge path.
136
158
  challengeWaitMs: z.number().step(100).min(0).max(MAX_CHALLENGE_WAIT_MS).default(DEFAULT_CHALLENGE_WAIT_MS).volatile(),
137
159
  challengeRetries: z.number().step(1).min(0).max(MAX_CHALLENGE_RETRIES).default(DEFAULT_CHALLENGE_RETRIES).volatile(),
160
+ // The deep-capture scroll walk's bounded budget; 0 disables the walk.
161
+ scrollBudgetMs: z.number().step(100).min(0).max(MAX_SCROLL_BUDGET_MS).default(DEFAULT_SCROLL_BUDGET_MS).volatile(),
138
162
  })
139
163
 
140
164
  /**
@@ -164,6 +188,8 @@ export interface PlaywrightPluginConfig {
164
188
  challengeWaitMs: Volatile<number>
165
189
  /** Challenge retry count as a live reference. */
166
190
  challengeRetries: Volatile<number>
191
+ /** Deep-capture scroll-walk budget (ms) as a live reference. */
192
+ scrollBudgetMs: Volatile<number>
167
193
  }
168
194
 
169
195
  /**
@@ -185,6 +211,7 @@ export function snapshotsOf(config: PlaywrightPluginConfig): Config {
185
211
  maxConcurrency: config.maxConcurrency.get(),
186
212
  challengeWaitMs: config.challengeWaitMs.get(),
187
213
  challengeRetries: config.challengeRetries.get(),
214
+ scrollBudgetMs: config.scrollBudgetMs.get(),
188
215
  }
189
216
  }
190
217
 
@@ -210,15 +237,19 @@ export function resolveConfig(config: Partial<Config>): ResolvedConfig {
210
237
  maxConcurrency: resolved.maxConcurrency.get(),
211
238
  challengeWaitMs: resolved.challengeWaitMs.get(),
212
239
  challengeRetries: resolved.challengeRetries.get(),
240
+ scrollBudgetMs: resolved.scrollBudgetMs.get(),
213
241
  }
214
242
  }
215
243
 
216
244
  /**
217
245
  * Complete config after schemastery applies the field defaults it owns.
218
- * `maxConcurrency` stays optional — {@link effectiveMaxConcurrency} resolves
219
- * it against the backend.
246
+ * `maxConcurrency` and `scrollBudgetMs` stay optional — the former because
247
+ * its effective default depends on `backend`, the latter so hand-built
248
+ * resolved configs (tests) may omit it and let
249
+ * {@link effectiveScrollBudgetMs} supply the default the schema would have.
220
250
  */
221
- export type ResolvedConfig = Omit<Required<Config>, 'maxConcurrency'> & { maxConcurrency?: number }
251
+ export type ResolvedConfig = Omit<Required<Config>, 'maxConcurrency' | 'scrollBudgetMs'>
252
+ & { maxConcurrency?: number; scrollBudgetMs?: number }
222
253
 
223
254
  /**
224
255
  * The concurrency limit a fetch actually runs with: an explicit setting
@@ -257,6 +288,18 @@ export function effectiveChallengeRetries(config: Pick<Config, 'challengeRetries
257
288
  return DEFAULT_CHALLENGE_RETRIES
258
289
  }
259
290
 
291
+ /**
292
+ * The scroll-walk budget a fetch actually runs with: an explicit setting
293
+ * wins, else the schema default. `0` means the deep-capture walk is off.
294
+ *
295
+ * @param config - the resolved settings section (or any partial of it).
296
+ * @returns the bounded walk budget in milliseconds.
297
+ */
298
+ export function effectiveScrollBudgetMs(config: Pick<Config, 'scrollBudgetMs'>): number {
299
+ if (typeof config.scrollBudgetMs === 'number') return config.scrollBudgetMs
300
+ return DEFAULT_SCROLL_BUDGET_MS
301
+ }
302
+
260
303
  /**
261
304
  * The context mode a fetch actually runs with: profile (a tab in the remote
262
305
  * browser's real profile) requires the CDP backend — a local launch has no
package/src/index.ts CHANGED
@@ -26,9 +26,12 @@ export {
26
26
  DEFAULT_CDP_ENDPOINT,
27
27
  DEFAULT_CHALLENGE_RETRIES,
28
28
  DEFAULT_CHALLENGE_WAIT_MS,
29
+ DEFAULT_SCROLL_BUDGET_MS,
30
+ MAX_SCROLL_BUDGET_MS,
29
31
  effectiveChallengeRetries,
30
32
  effectiveChallengeWaitMs,
31
33
  effectiveContextMode,
34
+ effectiveScrollBudgetMs,
32
35
  normalizeCdpEndpoint,
33
36
  resolveConfig,
34
37
  snapshotsOf,
@@ -45,6 +48,8 @@ export { CdpConnectionPool } from './cdp-pool.ts'
45
48
  export type { CdpAcquireMode, CdpConnect, CdpLease } from './cdp-pool.ts'
46
49
  export { CHALLENGE_DOM_PROBE, CHALLENGE_FINISH_RESERVE_MS, CHALLENGE_POLL_INTERVAL_MS, CHALLENGE_TITLE_RE, classifyChallengeHtml, classifyChallengeResponse, isChallengeCompatibleResponse } from './challenge.ts'
47
50
  export type { ChallengeVerdict } from './challenge.ts'
51
+ export { deepCaptureWalk } from './autoscroll.ts'
52
+ export type { DeepCapture, DeepCaptureStats } from './autoscroll.ts'
48
53
  export { PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider, WEB_FETCH_CHALLENGE_CODE } from './provider.ts'
49
54
  export { htmlToMarkdown } from './markdown.ts'
50
55
  export type { DenoiseMode, DenoiseResult } from './markdown.ts'
package/src/markdown.ts CHANGED
@@ -8,11 +8,14 @@
8
8
  * footers, and ad chrome by scoring link density and text mass) — its result
9
9
  * must then pass a coverage gate that distrusts tiny or sliver-thin
10
10
  * extractions, which on list-shaped pages (portals, catalogs) is exactly
11
- * what the link-density penalty leaves behind; DOMPurify sanitizes whatever
12
- * HTML remains and forbids the layout tags noise lives in; Turndown with
13
- * the GFM plugin converts to markdown using the same style options and
14
- * span-safe table rules as the shipped `dsh-tool-web` renderer, so output
15
- * is consistent with what `web_fetch` produces elsewhere.
11
+ * what the link-density penalty leaves behind; an accepted extraction is
12
+ * then pruned of trailing link-dense sections — the post-article
13
+ * recommendation rails and share bars the scorer's vocabularies cannot name
14
+ * (they arrive as bare divs with no class to match); DOMPurify sanitizes
15
+ * whatever HTML remains and forbids the layout tags noise lives in;
16
+ * Turndown with the GFM plugin converts to markdown using the same style
17
+ * options and span-safe table rules as the shipped `dsh-tool-web` renderer,
18
+ * so output is consistent with what `web_fetch` produces elsewhere.
16
19
  *
17
20
  * Synchronous for its callers and unit-tested against fixture pages: jsdom
18
21
  * itself is loaded lazily on the first call — and, inside a dsh profile, only
@@ -280,6 +283,245 @@ export interface DenoiseResult {
280
283
  mode: DenoiseMode
281
284
  }
282
285
 
286
+ /**
287
+ * Count non-whitespace characters under `root` that sit inside an anchor,
288
+ * skipping the same noise subtrees {@link visibleTextLength} skips so the
289
+ * numerator and denominator of a link-density ratio stay comparable.
290
+ *
291
+ * @param root - the subtree to measure; `null` counts as nothing.
292
+ * @returns the number of non-whitespace characters inside `<a>` descendants.
293
+ */
294
+ function linkTextLength(root: Node | null): number {
295
+ if (root === null) return 0
296
+ let length = 0
297
+ const visit = (node: Node, insideAnchor: boolean): void => {
298
+ if (node.nodeType === 3) {
299
+ if (insideAnchor) length += (node.nodeValue ?? '').replace(/\s+/g, '').length
300
+ return
301
+ }
302
+ if (node.nodeType !== 1 || NOISE_SUBTREE_TAGS.has(node.nodeName.toLowerCase())) return
303
+ const anchor = insideAnchor || node.nodeName.toLowerCase() === 'a'
304
+ for (let child = node.firstChild; child !== null; child = child.nextSibling) visit(child, anchor)
305
+ }
306
+ visit(root, false)
307
+ return length
308
+ }
309
+
310
+ /**
311
+ * Minimum non-whitespace characters a block must carry before its link
312
+ * density is even evaluated. Blocks below this are spacers, empty wrappers,
313
+ * or one-line sign-offs — too small to be worth a removal decision, so the
314
+ * trailing walk skips past them instead of stopping at them.
315
+ */
316
+ const TRAILING_MIN_TEXT_CHARS = 40
317
+
318
+ /**
319
+ * Link-density share above which a trailing block reads as a link farm:
320
+ * recommendation feeds, related-article rails, tag clouds, share bars.
321
+ * Measured on the 163.com dy-article badcase the section's pick carried 84
322
+ * linked characters inside 183 total (0.46) — recommendation cards dilute
323
+ * their density with author/date lines, so a 0.5 bar would miss them by
324
+ * construction; an honest closing paragraph with a citation or two measures
325
+ * 0.1–0.3, leaving 0.4 in the empty band between.
326
+ */
327
+ const TRAILING_LINK_DENSITY = 0.4
328
+
329
+ /**
330
+ * A trailing block qualifies as a link farm only when it is link-heavy in
331
+ * aggregate too: at least {@link TRAILING_LINK_COUNT} anchors or
332
+ * {@link TRAILING_LINK_TEXT_CHARS} linked characters. Density alone would
333
+ * let a single long linked sentence (an inline citation) drag a legitimate
334
+ * paragraph under the knife.
335
+ */
336
+ const TRAILING_LINK_COUNT = 3
337
+ const TRAILING_LINK_TEXT_CHARS = 60
338
+
339
+ /**
340
+ * A trailing block may never hold more than this share of the whole
341
+ * article's text. The prune targets *appended* widgets; a block that IS most
342
+ * of the extraction says the scorer picked the widget, which is the coverage
343
+ * gate's failure mode to handle, not this walk's.
344
+ */
345
+ const TRAILING_MAX_ARTICLE_SHARE = 0.6
346
+
347
+ /**
348
+ * A trailing run may never hold more than this share of its container's
349
+ * text — the aggregate form of the guard above, and the one that keeps
350
+ * list-shaped extractions whole. A recommendation rail is a *minority*
351
+ * appended to real content (the 163 rail is 44% of its container); a
352
+ * portal or feed whose extraction IS a list of link rows (measured on a
353
+ * Hacker News homepage capture: ~91 trailing rows, every one individually
354
+ * link-dense at 0.69–0.94) has a tail run holding ~97% of the container —
355
+ * there the tail IS the content, the walk declines wholesale, and only a
356
+ * run that is a minority of its container is removed.
357
+ */
358
+ const TRAILING_MAX_RUN_SHARE = 0.6
359
+
360
+ /**
361
+ * Link-density band within which a run and the block the walk stopped at
362
+ * read as the same material. When the content immediately before the run
363
+ * is itself as link-dense as the run — measured on a Hacker News homepage
364
+ * capture, the rows after the run's cut measure 0.94 against the run's
365
+ * 0.91 — the container is a *uniform list* (feed rows, portal headlines)
366
+ * and the tail is that list's own content, not a widget appended after it;
367
+ * a widget hangs after genuinely different material (the 163 rail's 0.44
368
+ * follows body text at 0.00). Only a run denser than what precedes it is
369
+ * removed.
370
+ */
371
+ const TRAILING_UNIFORM_DENSITY_BAND = 0.25
372
+
373
+ /**
374
+ * Prune trailing link-dense sections from an accepted Readability
375
+ * extraction — the post-article widget class the scorer itself cannot be
376
+ * taught to drop.
377
+ *
378
+ * Why this exists (badcase: `https://www.163.com/dy/article/…`): Chinese
379
+ * portal article pages append a recommendation feed inside the very wrapper
380
+ * Readability scores as the article (`#content` holds the body, the
381
+ * platform disclaimer, then a rail of image/title/author/date cards). Every
382
+ * existing noise layer misses it: Readability's `unlikelyCandidates` /
383
+ * `negative` vocabularies know `related` but not `recommend` (nor the
384
+ * pinyin families `tuijian`/`xiangguan`), and its own cleaning pass draws
385
+ * the "mostly links" line at per-block density 0.5 — these cards measure
386
+ * 0.42–0.48 (the title links over title+author+date text), so every card
387
+ * clears it; in the extracted HTML the rail is a *bare* div — class and id
388
+ * are gone — so no class-based rule can ever see it again; DOMPurify's
389
+ * FORBID_TAGS works on tags, and the rail is plain div/p/h3/a; and the
390
+ * coverage gate cannot help because the noise rides *inside* the
391
+ * extraction it measures (the badcase passed at 0.103 — the rail's text
392
+ * was in both numerator and denominator). Modern sites with CSS-module
393
+ * class names (`css-1b3x7r`) present the same shape with no vocabulary to
394
+ * match at all, so the only signal that survives all of this is
395
+ * structural: a *trailing* run of blocks whose text is mostly links.
396
+ *
397
+ * For every container in the extraction the walk gathers the maximal
398
+ * trailing run of qualifying children — link-farm blocks per the constants
399
+ * above, plus sub-threshold spacers (empty wrapper divs) — and removes the
400
+ * run only when BOTH structural guards say "appended widget, not content":
401
+ * the run is a minority of the container's text
402
+ * ({@link TRAILING_MAX_RUN_SHARE}), and the run is clearly denser than the
403
+ * block the walk stopped at ({@link TRAILING_UNIFORM_DENSITY_BAND}). The
404
+ * guards split the two worlds the per-block view cannot: a rail is
405
+ * one-or-few appended blocks after genuinely different material (the 163
406
+ * rail's 0.44 density follows body text at 0.00), while a link-list
407
+ * extraction — feed rows, portal headlines — is *uniform*: its rows before
408
+ * any cut measure the same density as the rows after it (0.91 vs 0.94 on
409
+ * the Hacker News capture), so no tail of it can be mistaken for a widget.
410
+ *
411
+ * Deliberately NOT applied to the whole-document fallback: that path
412
+ * exists for list-shaped pages where the links *are* the content (the
413
+ * 163.com homepage fix), and pruning link-dense regions there would
414
+ * re-introduce exactly the recall loss the coverage gate exists to
415
+ * prevent.
416
+ *
417
+ * @param container - parsed `article.content`; mutated in place.
418
+ */
419
+ function pruneTrailingLinkDenseSections(container: Element): void {
420
+ const articleText = visibleTextLength(container)
421
+ if (articleText === 0) return
422
+ // The container itself leads the walk — Readability usually wraps its
423
+ // content in div#readability-page-1, but an unwrapped extraction's
424
+ // top-level blocks are the container's direct children and must be
425
+ // examined too. querySelectorAll snapshots before any removal; entries
426
+ // inside an already-removed subtree are skipped by the contains() check,
427
+ // and parents are visited before children, so a deeper run inside a block
428
+ // an outer run is taking never competes with it.
429
+ for (const element of [container, ...Array.from(container.querySelectorAll('*'))]) {
430
+ if (!container.contains(element) && element !== container) continue
431
+ removeTrailingLinkDenseRun(element, articleText)
432
+ }
433
+ }
434
+
435
+ /**
436
+ * Remove one container's maximal trailing run of link-dense children when
437
+ * that run is a minority of the container's text AND clearly different
438
+ * material from the content that precedes it.
439
+ *
440
+ * @param parent - the container whose element children are examined from
441
+ * the end.
442
+ * @param articleText - the whole article's text, for the per-block share
443
+ * guard.
444
+ */
445
+ function removeTrailingLinkDenseRun(parent: Element, articleText: number): void {
446
+ const children = Array.from(parent.children)
447
+ if (children.length === 0) return
448
+ const containerText = visibleTextLength(parent)
449
+ if (containerText === 0) return
450
+ const run: Element[] = []
451
+ let runText = 0
452
+ let runLinked = 0
453
+ let stopIndex = -1
454
+ for (let index = children.length - 1; index >= 0; index--) {
455
+ const child = children[index]
456
+ if (child === undefined) break
457
+ const text = visibleTextLength(child)
458
+ // Too small to judge (spacer, empty wrapper): it belongs to the run
459
+ // but cannot extend it — keep walking toward earlier siblings.
460
+ if (text < TRAILING_MIN_TEXT_CHARS) {
461
+ run.push(child)
462
+ continue
463
+ }
464
+ const linked = linkTextLength(child)
465
+ const linkCount = child.querySelectorAll('a').length
466
+ if (
467
+ linked / text >= TRAILING_LINK_DENSITY
468
+ && (linkCount >= TRAILING_LINK_COUNT || linked >= TRAILING_LINK_TEXT_CHARS)
469
+ && text <= TRAILING_MAX_ARTICLE_SHARE * articleText
470
+ ) {
471
+ run.push(child)
472
+ runText += text
473
+ runLinked += linked
474
+ continue
475
+ }
476
+ // Real content that is not a link farm: everything before it is
477
+ // anchored by actual article flow — stop and compare against it.
478
+ stopIndex = index
479
+ break
480
+ }
481
+ if (runText === 0 || stopIndex === -1) {
482
+ // Nothing to remove — or the run reached the top of the container (no
483
+ // preceding content to differ from): a tail that IS the whole
484
+ // container is list content, never a widget.
485
+ return
486
+ }
487
+ // The tail is the same material as what precedes it (a uniform list's
488
+ // rows): decline wholesale.
489
+ const stopChild = children[stopIndex]
490
+ if (stopChild === undefined) return
491
+ const stopDensity = linkTextLength(stopChild) / Math.max(1, visibleTextLength(stopChild))
492
+ if (Math.abs(runLinked / runText - stopDensity) <= TRAILING_UNIFORM_DENSITY_BAND) return
493
+ // The tail holds most of the container (a link-list page's rows): decline
494
+ // wholesale.
495
+ if (runText > TRAILING_MAX_RUN_SHARE * containerText) return
496
+ for (const child of run) child.remove()
497
+ }
498
+
499
+ /**
500
+ * Run {@link pruneTrailingLinkDenseSections} over one extraction with a
501
+ * misfire guard: a prune that takes a healthy article below the gate's
502
+ * absolute floor ({@link GATE_MIN_ARTICLE_TEXT_CHARS}) or empties it
503
+ * entirely says the walk misread the page, and Readability's original pick
504
+ * is returned untouched. A *sparse* article (under the floor before the
505
+ * prune, auto-trusted by the gate) keeps its pruned form — its trailing
506
+ * links were noise all the same.
507
+ *
508
+ * @param articleHtml - the `article.content` HTML Readability produced.
509
+ * @param document - the live document, for parsing the fragment.
510
+ * @returns the pruned article HTML, or the original on a misfire.
511
+ */
512
+ function prunedArticleContent(articleHtml: string, document: Document): string {
513
+ const container = document.createElement('div')
514
+ container.innerHTML = articleHtml
515
+ const before = visibleTextLength(container)
516
+ if (before === 0) return articleHtml
517
+ pruneTrailingLinkDenseSections(container)
518
+ const after = visibleTextLength(container)
519
+ if (after === 0 || (before >= GATE_MIN_ARTICLE_TEXT_CHARS && after < GATE_MIN_ARTICLE_TEXT_CHARS)) {
520
+ return articleHtml
521
+ }
522
+ return container.innerHTML
523
+ }
524
+
283
525
  /**
284
526
  * Convert one rendered HTML document to denoised markdown.
285
527
  *
@@ -323,7 +565,11 @@ export function htmlToMarkdown(html: string, url: string): DenoiseResult {
323
565
  article !== null && typeof article.content === 'string' && article.content !== ''
324
566
  && articleCoversPage(article.content, document)
325
567
  ) {
326
- source = article.content
568
+ // Gate first, prune second, and in that order on purpose: the gate
569
+ // must judge Readability's pick as delivered (the widget rides inside
570
+ // it), and the prune then strips the post-article link farms the
571
+ // scorer could not be taught — see pruneTrailingLinkDenseSections.
572
+ source = prunedArticleContent(article.content, document)
327
573
  title = article.title ?? undefined
328
574
  }
329
575
  } catch {
package/src/provider.ts CHANGED
@@ -50,6 +50,31 @@
50
50
  * fails with {@link WEB_FETCH_CHALLENGE_CODE} instead of returning the
51
51
  * interstitial as content.
52
52
  *
53
+ * Deep capture (viewport-virtualized feeds and app-shell documents): feed-
54
+ * shaped pages — Discourse topic streams, comment threads, timelines —
55
+ * render only the items near the scroll position; everything else is an
56
+ * empty, height-only placeholder that keeps the item's id but none of its
57
+ * content, so a single-instant capture hands back the skeleton while the
58
+ * bodies silently vanish (and the denoise pipeline's coverage gate cannot
59
+ * notice: the page's own visible text is the same windowed fragment as the
60
+ * extraction). App-shell document editors (Feishu docs) push the same
61
+ * pathology one level down: the body never scrolls, blocks are keyed by
62
+ * data-block-id with no id, and off-window blocks are REMOVED behind an
63
+ * anonymous height spacer — the fetch would return just the statically
64
+ * rendered first screen. Before serialization the fetch therefore runs one
65
+ * bounded in-page scroll walk — over the window or, when the document does
66
+ * not scroll, the page's dominant inner scroll container — stashing what
67
+ * each rendered window shows, terminating only AT the bottom with the
68
+ * height stable (that is what pages load-more sentinels; height stability
69
+ * elsewhere proves nothing), then rehydrating degraded keyed elements from
70
+ * their stashed snapshots and splicing removed keys back at their document
71
+ * position so the final DOM holds the union of every window in document
72
+ * order. The existing pipeline then runs unchanged. Bounded by
73
+ * `scrollBudgetMs` (default 20s; 0 restores the legacy single-instant
74
+ * behavior) and by the remaining fetch deadline; any walk failure silently
75
+ * degrades to the legacy capture (see `src/autoscroll.ts` for the measured
76
+ * linux.do and Feishu badcases).
77
+ *
53
78
  * @module dsh-web-fetch-playwright/provider
54
79
  */
55
80
 
@@ -57,10 +82,12 @@ import { WebError } from '@deepseek-ai/dsh-web'
57
82
  import type { WebFetchProvider, WebFetchRequest, WebFetchResult } from '@deepseek-ai/dsh-web'
58
83
  import { CHALLENGE_DOM_PROBE, CHALLENGE_FINISH_RESERVE_MS, CHALLENGE_POLL_INTERVAL_MS, classifyChallengeHtml, classifyChallengeResponse, isChallengeCompatibleResponse } from './challenge.ts'
59
84
  import type { ChallengeVerdict } from './challenge.ts'
60
- import { DEFAULT_MAX_CONCURRENCY_CDP, DEFAULT_MAX_CONCURRENCY_LOCAL, effectiveChallengeRetries, effectiveChallengeWaitMs, effectiveContextMode, effectiveMaxConcurrency, normalizeCdpEndpoint } from './config.ts'
85
+ import { deepCaptureWalk, SCROLL_FINISH_RESERVE_MS } from './autoscroll.ts'
86
+ import { DEFAULT_MAX_CONCURRENCY_CDP, DEFAULT_MAX_CONCURRENCY_LOCAL, effectiveChallengeRetries, effectiveChallengeWaitMs, effectiveContextMode, effectiveMaxConcurrency, effectiveScrollBudgetMs, normalizeCdpEndpoint } from './config.ts'
61
87
  import type { ResolvedConfig } from './config.ts'
62
88
  import { CdpConnectionPool } from './cdp-pool.ts'
63
89
  import { htmlToMarkdown } from './markdown.ts'
90
+ import { STRIP_SCRIPTING_ELEMENTS } from './strip.ts'
64
91
  import { resolveCdpBackend, resolvePlaywrightBackend } from './playwright-resolve.ts'
65
92
  import type { PlaywrightBrowser, PlaywrightContext, PlaywrightDownload, PlaywrightPage, PlaywrightResponse, PlaywrightRoute } from './types.ts'
66
93
 
@@ -535,14 +562,31 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
535
562
  // what domcontentloaded already produced.
536
563
  await page.waitForLoadState('networkidle', { timeout: Math.min(SETTLE_MS, deadline.remainingMs()) }).catch(() => {})
537
564
 
538
- // Serialization safety: the document is about to be flattened into an
539
- // HTML string and re-parsed by jsdom, and scripting elements are the one
540
- // place that round-trip can break — see {@link STRIP_SCRIPTING_ELEMENTS}
541
- // for exactly what is removed, what is kept (JSON-LD, noscript), and why
542
- // each removal cannot change what the pipeline extracts.
543
- await stripScriptingElements(page)
565
+ // Deep capture (viewport-virtualized feeds and app-shell documents lose
566
+ // their off-window content at rest — see `src/autoscroll.ts`): a bounded
567
+ // walk stashes what each rendered window shows and, when it restored
568
+ // anything, hands back its own merged, already-stripped serialization.
569
+ // HTML documents only, inside the deadline with a decode reserve,
570
+ // silently degrading to the legacy capture; `scrollBudgetMs: 0` skips it.
571
+ const scrollBudgetMs = Math.min(
572
+ effectiveScrollBudgetMs(config),
573
+ deadline.remainingMs() - SCROLL_FINISH_RESERVE_MS,
574
+ )
575
+ const captured = scrollBudgetMs > 0 ? await deepCaptureWalk(page, scrollBudgetMs) : undefined
544
576
 
545
- const html = await page.content()
577
+ // Serialization safety for the live read: the document is about to be
578
+ // flattened into an HTML string and re-parsed by jsdom, and scripting
579
+ // elements are the one place that round-trip can break — see
580
+ // `src/strip.ts` for exactly what is removed, what is kept (JSON-LD,
581
+ // noscript), and why each removal cannot change what the pipeline
582
+ // extracts. (The walk's merged capture went through the same strip.)
583
+ let html: string
584
+ if (captured?.html !== undefined) {
585
+ html = captured.html
586
+ } else {
587
+ await stripScriptingElements(page)
588
+ html = await page.content()
589
+ }
546
590
  if (!config.denoise) {
547
591
  // The tool layer's own turndown renders raw HTML; the checkbox only
548
592
  // governs the Readability/DOMPurify stage this provider owns.
@@ -907,68 +951,13 @@ async function probeStillOnChallenge(page: PlaywrightPage): Promise<boolean> {
907
951
  }
908
952
  }
909
953
 
910
- /**
911
- * In-page script that removes non-content scripting subtrees from the live
912
- * document before `page.content()` reads it as an HTML string.
913
- *
914
- * WHAT goes, and why each removal is output-neutral versus the old pipeline:
915
- *
916
- * - `script` — three consumers ever saw script text before, and none loses
917
- * it here: Readability's `parse()` deletes script/noscript nodes BEFORE
918
- * extraction and scoring (so script text never reached the article or
919
- * influenced the scorer); DOMPurify's default allowlist drops script; the
920
- * tool layer's renderer removes SCRIPT wholesale on the raw path. What the
921
- * removal buys is the reason this exists: a page's runtime JS may rewrite
922
- * a data script's text with a literal `</script>` embedded — measured on
923
- * techcommunity.microsoft.com (Khoros/Aurora), whose client writes its
924
- * Apollo cache back into `script#__NEXT_DATA__` via `JSON.stringify`,
925
- * turning the wire's 712 `\u003c` escapes back into raw `<` — and
926
- * browsers serialize script text verbatim, so the flattened string ends
927
- * that script at the FIRST embedded end tag and re-parses the remaining
928
- * ~270KB of JSON as markup. The phantom elements that survive (unknown
929
- * tags like `<id>`, anchors full of JSON text) are exactly what
930
- * Readability's scorer reads as article mass, so the extracted "article"
931
- * is CSS and i18n strings while the real body — still present in the DOM —
932
- * loses. No string-level patch can undo this after serialization (the
933
- * string is genuinely ambiguous); only the browser's own parse holds the
934
- * document's true structure.
935
- * - `script[type="application/ld+json"]` — KEPT, but defused: Readability
936
- * extracts metadata (title/byline/excerpt — e.g. the site-suffix-free
937
- * "Remote Desktop Protocol" instead of "… - Wikipedia") from JSON-LD
938
- * scripts before removing them, so deleting them would regress the title
939
- * on every JSON-LD-carrying page. Rewriting `</` as `<\/` keeps the text
940
- * valid JSON (`\/` is a legal escape, so the re-parsed text parses to the
941
- * same object) while making the serialization harmless: the tokenizer
942
- * only terminates on a literal `</script`.
943
- * - `noscript` — KEPT: Readability's `_unwrapNoscriptImages` salvages the
944
- * real image out of lazy-load `<noscript><img …>` wrappers before
945
- * `_removeScripts` deletes the noscript; pre-stripping it would silently
946
- * drop those images from the markdown.
947
- * - `template` — removed: its children parse into the inert `.content`
948
- * fragment (invisible to Readability's traversal), DOMPurify's
949
- * FORBID_TAGS drops it, and the tool layer's renderer removes it too.
950
- * - `style` stays — its text cannot terminate a re-parse except at its own
951
- * end tag, and the pipeline already drops it.
952
- */
953
- const STRIP_SCRIPTING_ELEMENTS = `(() => {
954
- for (const el of Array.from(document.querySelectorAll('script, template'))) {
955
- if (el.tagName === 'SCRIPT' && (el.getAttribute('type') || '').trim().toLowerCase() === 'application/ld+json') {
956
- // '<\\/' is still valid JSON, so the re-parsed textContent parses to
957
- // the same object — while the serialized form can no longer close the
958
- // script element early.
959
- el.textContent = el.textContent.replace(/<\\//g, '<\\\\/')
960
- continue
961
- }
962
- el.remove()
963
- }
964
- })()`
965
-
966
954
  /**
967
955
  * Remove non-content scripting subtrees from the live document before
968
- * serialization. Best-effort, mirroring {@link probeStillOnChallenge}: a
969
- * backend without `evaluate` (fakes, minimal engines) simply keeps the
970
- * document as-is — the fetch never fails here, and the pipeline degrades to
971
- * whatever the serialized string itself says.
956
+ * serialization (see `stripScriptingElementsIn` in `src/strip.ts` for exactly
957
+ * what goes and why). Best-effort, mirroring {@link probeStillOnChallenge}: a backend
958
+ * without `evaluate` (fakes, minimal engines) simply keeps the document
959
+ * as-is — the fetch never fails here, and the pipeline degrades to whatever
960
+ * the serialized string itself says.
972
961
  */
973
962
  async function stripScriptingElements(page: PlaywrightPage): Promise<void> {
974
963
  if (typeof page.evaluate !== 'function') return