dsh-web-fetch-playwright 0.2.15 → 0.2.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/index.js CHANGED
@@ -9,6 +9,17 @@ import { fileURLToPath, pathToFileURL } from "node:url";
9
9
  import { accessSync, constants, readFileSync, realpathSync, statSync } from "node:fs";
10
10
  import { delimiter, dirname, join } from "node:path";
11
11
  //#region src/config.ts
12
+ /**
13
+ * Default bounded budget (ms) for the deep-capture scroll walk
14
+ * (`src/autoscroll.ts`). Sized so long topics stay fully covered at the
15
+ * careful cadence (300ms motion + 300ms rest per ~0.7-viewport step, plus
16
+ * the bottom patience window): 20s walks ~25kpx with pagination, covering
17
+ * the measured linux.do badcase (~18kpx, 51 replies) with headroom.
18
+ * Scroll-inert pages take the walk's fast path and spend a few seconds.
19
+ */
20
+ const DEFAULT_SCROLL_BUDGET_MS = 2e4;
21
+ /** Ceiling the schema accepts for `scrollBudgetMs`. */
22
+ const MAX_SCROLL_BUDGET_MS = 3e4;
12
23
  /** Default CDP endpoint when the settings section leaves it blank. */
13
24
  const DEFAULT_CDP_ENDPOINT = "127.0.0.1:9222";
14
25
  /**
@@ -31,7 +42,8 @@ const Config = z.object({
31
42
  denoise: z.boolean().default(true).volatile(),
32
43
  maxConcurrency: z.number().step(1).min(1).max(200).volatile(),
33
44
  challengeWaitMs: z.number().step(100).min(0).max(MAX_CHALLENGE_WAIT_MS).default(DEFAULT_CHALLENGE_WAIT_MS).volatile(),
34
- challengeRetries: z.number().step(1).min(0).max(3).default(1).volatile()
45
+ challengeRetries: z.number().step(1).min(0).max(3).default(1).volatile(),
46
+ scrollBudgetMs: z.number().step(100).min(0).max(MAX_SCROLL_BUDGET_MS).default(DEFAULT_SCROLL_BUDGET_MS).volatile()
35
47
  });
36
48
  /**
37
49
  * Read the live references `apply` holds into one plain input for
@@ -51,7 +63,8 @@ function snapshotsOf(config) {
51
63
  denoise: config.denoise.get(),
52
64
  maxConcurrency: config.maxConcurrency.get(),
53
65
  challengeWaitMs: config.challengeWaitMs.get(),
54
- challengeRetries: config.challengeRetries.get()
66
+ challengeRetries: config.challengeRetries.get(),
67
+ scrollBudgetMs: config.scrollBudgetMs.get()
55
68
  };
56
69
  }
57
70
  /**
@@ -75,7 +88,8 @@ function resolveConfig(config) {
75
88
  denoise: resolved.denoise.get(),
76
89
  maxConcurrency: resolved.maxConcurrency.get(),
77
90
  challengeWaitMs: resolved.challengeWaitMs.get(),
78
- challengeRetries: resolved.challengeRetries.get()
91
+ challengeRetries: resolved.challengeRetries.get(),
92
+ scrollBudgetMs: resolved.scrollBudgetMs.get()
79
93
  };
80
94
  }
81
95
  /**
@@ -113,6 +127,17 @@ function effectiveChallengeRetries(config) {
113
127
  return 1;
114
128
  }
115
129
  /**
130
+ * The scroll-walk budget a fetch actually runs with: an explicit setting
131
+ * wins, else the schema default. `0` means the deep-capture walk is off.
132
+ *
133
+ * @param config - the resolved settings section (or any partial of it).
134
+ * @returns the bounded walk budget in milliseconds.
135
+ */
136
+ function effectiveScrollBudgetMs(config) {
137
+ if (typeof config.scrollBudgetMs === "number") return config.scrollBudgetMs;
138
+ return DEFAULT_SCROLL_BUDGET_MS;
139
+ }
140
+ /**
116
141
  * The context mode a fetch actually runs with: profile (a tab in the remote
117
142
  * browser's real profile) requires the CDP backend — a local launch has no
118
143
  * meaningful shared profile — and the share option (which defaults on, so an
@@ -161,14 +186,22 @@ function normalizeCdpEndpoint(input) {
161
186
  * 2. Content-level — the interstitial's localized `<title>` family
162
187
  * ("Just a moment...", "请稍候…", ...) plus structural markers
163
188
  * (`/cdn-cgi/challenge-platform/` scripts, `#challenge-*` elements,
164
- * `cf-chl-widget-` Turnstile frames, `window._cf_chl_opt`). Markers are
165
- * deliberately structural: a blog post that merely MENTIONS Cloudflare
166
- * must not classify as a challenge, so the challenge-platform script
167
- * check reads `<script src>` attributes only (never prose or escaped
168
- * code samples) and identifies the runtime POSITIVELY by its
169
- * `/orchestrate/` mount — the JSD/Precursor instrumentation Cloudflare
170
- * injects into every NORMAL page of a protected zone, on several moving
171
- * mounts, is not a challenge signal.
189
+ * `window._cf_chl_opt`). Markers are deliberately structural: a blog
190
+ * post that merely MENTIONS Cloudflare must not classify as a challenge,
191
+ * so the challenge-platform script check reads `<script src>` attributes
192
+ * only (never prose or escaped code samples) and identifies the runtime
193
+ * POSITIVELY by its `/orchestrate/` mount — the JSD/Precursor
194
+ * instrumentation Cloudflare injects into every NORMAL page of a
195
+ * protected zone, on several moving mounts, is not a challenge signal.
196
+ * The Turnstile widget id (`cf-chl-widget-…`) is deliberately NOT a
197
+ * marker: sites embed real Turnstile widgets into normal pages
198
+ * (linux.do — a Discourse forum with the Turnstile plugin mounts one on
199
+ * every topic page), so the widget alone cannot distinguish an
200
+ * interstitial from a normal page behind a Cloudflare edge. Every real
201
+ * interstitial carries at least one independent signal beyond its
202
+ * widget — the localized title, the orchestrate runtime script,
203
+ * `window._cf_chl_opt`, or the `#challenge-*` scaffolding — and those
204
+ * stay sufficient on their own (see {@link CHALLENGE_HTML_MARKERS}).
172
205
  *
173
206
  * The security stance (issue #2): this module only DETECTS and classifies.
174
207
  * Solving, spoofing, CAPTCHA answering, and cookie lifting live outside the
@@ -314,9 +347,21 @@ function isChallengeRuntimeScript(src) {
314
347
  * The challenge-platform prefix is deliberately absent: script evidence goes
315
348
  * through {@link scriptSrcsOf} + {@link isChallengeRuntimeScript}, which
316
349
  * distinguish a real `src` from prose and runtime from instrumentation.
350
+ *
351
+ * The Turnstile widget id (`id="cf-chl-widget-…"`) is deliberately ABSENT
352
+ * (badcase: linux.do/t/topic/3143588 — a Discourse forum behind Cloudflare
353
+ * whose Turnstile plugin mounts a real widget, id and all, into every normal
354
+ * topic page it serves at HTTP 200). The widget is minted by the same
355
+ * Turnstile runtime whether it renders inside an interstitial or inside a
356
+ * site's own UI, so its presence proves only "this page embeds Turnstile".
357
+ * On a real interstitial the widget always co-occurs with independent proof
358
+ * — the localized title, the orchestrate runtime script,
359
+ * `window._cf_chl_opt`, or the `#challenge-*` scaffolding, each sufficient
360
+ * above — so dropping the widget loses no real interstitial while every
361
+ * widget-embedding normal page (linux.do, login portals, contact forms)
362
+ * stops dying as WEB_FETCH_CHALLENGE after a wait that can never clear.
317
363
  */
318
364
  const CHALLENGE_HTML_MARKERS = [
319
- "id=\"cf-chl-widget-",
320
365
  "window._cf_chl_opt",
321
366
  "id=\"challenge-form\"",
322
367
  "id=\"challenge-running\"",
@@ -410,7 +455,13 @@ function classifyChallengeHtml(html) {
410
455
  * check uses, so the two can never drift apart. Script evidence follows the
411
456
  * same rule as {@link classifyChallengeHtml}: only the challenge runtime
412
457
  * counts, and the instrumentation Cloudflare injects into normal pages of a
413
- * protected zone is ignored.
458
+ * protected zone is ignored. The Turnstile widget selector is likewise
459
+ * absent for the same reason {@link CHALLENGE_HTML_MARKERS} omits it: a
460
+ * widget embedded in a NORMAL page (linux.do's Turnstile plugin) would keep
461
+ * the probe reporting "still challenged" forever, so a challenge that HAD
462
+ * cleared could never be confirmed — the wait burned its whole budget and
463
+ * the fetch died as WEB_FETCH_CHALLENGE (last status 200) or the tool
464
+ * layer's timeout.
414
465
  */
415
466
  const CHALLENGE_DOM_PROBE = `(() => {
416
467
  const re = new RegExp(${JSON.stringify(CHALLENGE_TITLE_RE.source)}, ${JSON.stringify(CHALLENGE_TITLE_RE.flags)})
@@ -418,7 +469,6 @@ const CHALLENGE_DOM_PROBE = `(() => {
418
469
  const runtime = ${JSON.stringify(CHALLENGE_RUNTIME_SEGMENT)}
419
470
  if (re.test(document.title.trim())) return true
420
471
  if (document.querySelector('#challenge-form, #challenge-running, #challenge-stage, #challenge-error-text')) return true
421
- if (document.querySelector('[id^="cf-chl-widget-"]')) return true
422
472
  for (const script of Array.from(document.scripts)) {
423
473
  const src = (script.getAttribute('src') || '').toLowerCase()
424
474
  if (src.includes(platform) && src.includes(runtime)) return true
@@ -427,6 +477,593 @@ const CHALLENGE_DOM_PROBE = `(() => {
427
477
  return false
428
478
  })()`;
429
479
  //#endregion
480
+ //#region src/strip.ts
481
+ /**
482
+ * Serialization safety for documents about to be flattened into an HTML
483
+ * string and re-parsed by jsdom: the one in-page cleanup both serialization
484
+ * paths share — the provider's live-document read and the deep-capture
485
+ * walk's merged clone.
486
+ *
487
+ * @module dsh-web-fetch-playwright/strip
488
+ */
489
+ /**
490
+ * In-page function that removes non-content scripting subtrees from a
491
+ * document before it is flattened into an HTML string — the live document
492
+ * before `page.content()`, or the deep-capture walk's merged clone before it
493
+ * serializes it itself (`src/autoscroll.ts`).
494
+ *
495
+ * It is evaluated BY SOURCE (`fn.toString()`), so it must stay
496
+ * self-contained: no imports, no module-level references.
497
+ *
498
+ * WHAT goes, and why each removal is output-neutral versus the old pipeline:
499
+ *
500
+ * - `script` — three consumers ever saw script text before, and none loses
501
+ * it here: Readability's `parse()` deletes script/noscript nodes BEFORE
502
+ * extraction and scoring (so script text never reached the article or
503
+ * influenced the scorer); DOMPurify's default allowlist drops script; the
504
+ * tool layer's renderer removes SCRIPT wholesale on the raw path. What the
505
+ * removal buys is the reason this exists: a page's runtime JS may rewrite
506
+ * a data script's text with a literal `<\/script>` embedded — measured on
507
+ * techcommunity.microsoft.com (Khoros/Aurora), whose client writes its
508
+ * Apollo cache back into `script#__NEXT_DATA__` via `JSON.stringify`,
509
+ * turning the wire's 712 `\u003c` escapes back into raw `<` — and
510
+ * browsers serialize script text verbatim, so the flattened string ends
511
+ * that script at the FIRST embedded end tag and re-parses the remaining
512
+ * ~270KB of JSON as markup. The phantom elements that survive (unknown
513
+ * tags like `<id>`, anchors full of JSON text) are exactly what
514
+ * Readability's scorer reads as article mass, so the extracted "article"
515
+ * is CSS and i18n strings while the real body — still present in the DOM —
516
+ * loses. No string-level patch can undo this after serialization (the
517
+ * string is genuinely ambiguous); only the browser's own parse holds the
518
+ * document's true structure.
519
+ * - `script[type="application/ld+json"]` — KEPT, but defused: Readability
520
+ * extracts metadata (title/byline/excerpt — e.g. the site-suffix-free
521
+ * "Remote Desktop Protocol" instead of "… - Wikipedia") from JSON-LD
522
+ * scripts before removing them, so deleting them would regress the title
523
+ * on every JSON-LD-carrying page. Rewriting `</` as `<\/` keeps the text
524
+ * valid JSON (`\/` is a legal escape, so the re-parsed text parses to the
525
+ * same object) while making the serialization harmless: the tokenizer
526
+ * only terminates on a literal `<\/script`.
527
+ * - `noscript` — KEPT: Readability's `_unwrapNoscriptImages` salvages the
528
+ * real image out of lazy-load `<noscript><img …>` wrappers before
529
+ * `_removeScripts` deletes the noscript; pre-stripping it would silently
530
+ * drop those images from the markdown.
531
+ * - `template` — removed: its children parse into the inert `.content`
532
+ * fragment (invisible to Readability's traversal), DOMPurify's
533
+ * FORBID_TAGS drops it, and the tool layer's renderer removes it too.
534
+ * - `style` stays — its text cannot terminate a re-parse except at its own
535
+ * end tag, and the pipeline already drops it.
536
+ */
537
+ function stripScriptingElementsIn(doc) {
538
+ for (const el of Array.from(doc.querySelectorAll("script, template"))) {
539
+ if (el.tagName === "SCRIPT" && (el.getAttribute("type") ?? "").trim().toLowerCase() === "application/ld+json") {
540
+ el.textContent = (el.textContent ?? "").replace(/<\//g, "<\\/");
541
+ continue;
542
+ }
543
+ el.remove();
544
+ }
545
+ }
546
+ /** {@link stripScriptingElementsIn} over the live document, as an evaluate expression. */
547
+ const STRIP_SCRIPTING_ELEMENTS = `(${stripScriptingElementsIn.toString()})(document)`;
548
+ //#endregion
549
+ //#region src/autoscroll.ts
550
+ /**
551
+ * Deep capture: a bounded in-page scroll walk that defeats viewport
552
+ * virtualization before the document is serialized.
553
+ *
554
+ * WHY. Long "feed-shaped" pages — Discourse topic streams (linux.do,
555
+ * meta.discourse.org), comment threads, timelines — increasingly render only
556
+ * the posts near the current scroll position. Discourse is the measured
557
+ * badcase: at rest the live DOM holds the posts within roughly one viewport
558
+ * of the top, while every post outside that window is a height-only
559
+ * placeholder that keeps the post's identity but none of its content:
560
+ *
561
+ * ```html
562
+ * <div class="post-stream--cloaked" data-post-number="1" id="post_1"
563
+ * style="height:297.9px !important;…">
564
+ * <h2 class="sr-only">由 Diver-QC 于 2 小时前 发布</h2>
565
+ * </div>
566
+ * ```
567
+ *
568
+ * The placeholder carries the SAME id the rendered `<article id="post_1">`
569
+ * has — every mainstream virtualized list needs some stable identity to
570
+ * recycle its rows. A fetch that renders, never scrolls, and serializes one
571
+ * instant of that DOM returns the skeleton (the screen-reader-only "posted
572
+ * by" headings survive the pipeline as if they were content) while the reply
573
+ * bodies silently vanish. Scrolling to the bottom once does not help either:
574
+ * the window MOVES — measured on the same page, scrolling to the bottom left
575
+ * only posts 31–40 rendered and re-cloaked 1–30. And the denoise pipeline's
576
+ * own coverage gate cannot notice the loss: the page's "visible text" it
577
+ * measures against is the same windowed fragment as the extraction, so the
578
+ * ratio always passes. Only walk depth can see what a resting viewport hides.
579
+ *
580
+ * The SAME pathology, one level deeper, hits app-shell document editors —
581
+ * the second measured badcase is a Feishu wiki page (bocha-ai.feishu.cn).
582
+ * There the body never scrolls at all: html/body are locked at exactly one
583
+ * viewport and the document scrolls inside `div.bear-web-x-container`
584
+ * (measured: documentElement.scrollHeight 720 == innerHeight,
585
+ * window.scrollTo(0, 800) left scrollY at 0, while the container held 4534px
586
+ * against a 656px client). Its blocks are virtualized too, with two twists
587
+ * over Discourse: blocks carry `data-block-id` and NO id, and off-window
588
+ * blocks are REMOVED outright — the whole un-rendered tail is one anonymous
589
+ * `.bear-virtual-renderUnit-placeholder` spacer with an explicit height — so
590
+ * there is neither a window scrollbar to walk nor a degraded-in-place row to
591
+ * swap back. A fetch therefore returns the statically rendered first screen.
592
+ *
593
+ * HOW. Between the settle and the serialization the provider runs one
594
+ * in-page async walk ({@link deepCaptureInPage}, a typed function evaluated
595
+ * by source):
596
+ *
597
+ * 1. Resolve the walk TARGET: the window when the document itself scrolls
598
+ * (the Discourse shape), else the page's dominant inner scroll container
599
+ * (the app-shell shape) — the element with the largest scrollable
600
+ * distance among computed `overflow-y: auto|scroll` elements, gated at
601
+ * half a viewport so dropdown-sized scrollers never trigger a walk. A
602
+ * page with neither is left alone (the legacy capture).
603
+ * 2. STASH every keyed content element — key = id, else a namespaced
604
+ * `data-block-id` — as a detached clone (the RICHEST sighting per key
605
+ * wins), together with its nearest keyed siblings: where it sits in its
606
+ * own list, which is what a removed row is later spliced back next to.
607
+ * 3. WALK down from the top in smooth 0.7-viewport steps (300ms motion +
608
+ * 300ms quiet settle — the rest a feed engine's debounced uncloak needs),
609
+ * collecting after each, until the walk can hold AT THE BOTTOM through a
610
+ * {@link SCROLL_BOTTOM_PATIENCE_MS} window with the height unchanged
611
+ * (growth restarts the clock — that is what pages load-more sentinels),
612
+ * or the step/budget caps close it.
613
+ *
614
+ * A MutationObserver watches for the walk's whole reason to exist: the
615
+ * page REACTING to scroll by adding or removing elements. Most of the web
616
+ * never does (a long static article only costs latency here), so after
617
+ * {@link SCROLL_INERT_STEPS} careful steps with no reaction the walk
618
+ * probes the rest with one jump to the bottom. Any reaction after that
619
+ * jump — an uncloak, a re-cloak, a load-more append — means the region
620
+ * jumped over may hold content that never rendered, so the walk rewinds
621
+ * to where it jumped from and finishes carefully. Correctness never
622
+ * rests on the warm-up guess; only the cost of a scroll-inert page does.
623
+ * 4. MERGE into a structural clone (`document.cloneNode(true)`: no
624
+ * serialize/re-parse round trip — a runtime-rewritten data script with a
625
+ * literal `<\/script>` in its text cannot corrupt it — and, having no
626
+ * browsing context, it runs no scripts, upgrades no custom elements and
627
+ * loads no frames):
628
+ * - SPLICE (container walks): stashed keys from inside the scroll
629
+ * container that the clone no longer holds were REMOVED by the engine
630
+ * (the Feishu shape — its placeholders are anonymous) and go back next
631
+ * to their recorded sibling, resolved iteratively so a removed run
632
+ * chains off whichever end survived; a key that rode in inside its
633
+ * spliced parent is already present and skipped. Placement is
634
+ * structural, so it is right whatever order the walk SAW rows in (an
635
+ * anchored open starts mid-document).
636
+ * - SWAP: every keyed element emptier than its snapshot (fewer child
637
+ * elements, or meaningfully less text — the Discourse placeholder) is
638
+ * replaced by that snapshot, except containers (ancestors of another
639
+ * element being swapped), which are assembled from those descendants
640
+ * instead: a container's own snapshot is one window's partial view.
641
+ * The clone is then stripped of scripting elements and serialized; the
642
+ * provider reads that string instead of the live page. The live document
643
+ * is never rewritten, so no late scroll/rAF/timer reaction can touch the
644
+ * capture either.
645
+ *
646
+ * Failure is always silent degradation: a page without `evaluate`, a script
647
+ * error, or an engine that rejects the walk just keeps the legacy
648
+ * single-instant capture — a walk can only add content, never fail a fetch.
649
+ * A walk that restored nothing returns no HTML, and the provider reads the
650
+ * live page exactly as before.
651
+ *
652
+ * The walk runs in BOTH denoise modes (the raw-HTML path serializes the same
653
+ * cloaked placeholders) but never for non-HTML documents. It is bounded by
654
+ * the `scrollBudgetMs` setting (`0` restores the pre-walk legacy behavior)
655
+ * and always by the remaining fetch deadline minus a decode reserve — the
656
+ * provider's abort closes the tab, which rejects the pending evaluation.
657
+ *
658
+ * @module dsh-web-fetch-playwright/autoscroll
659
+ */
660
+ /** How far each step scrolls, as a share of the viewport height. */
661
+ const SCROLL_STEP_RATIO = .7;
662
+ /**
663
+ * How long at the bottom, with no height growth, before the walk concludes
664
+ * the page is complete. Real feed engines fetch their next chunk with
665
+ * network latency (measured on linux.do: a pending chunk landed well after
666
+ * two 450ms checks), so this is sized over a slow round trip.
667
+ */
668
+ const SCROLL_BOTTOM_PATIENCE_MS = 2500;
669
+ /**
670
+ * Cap on stashed keys — pathological id counts cannot pin memory. Sized for
671
+ * long topics: a Discourse post carries several ids of its own (post,
672
+ * heading, avatar, …), so a few hundred posts already need a thousand-plus.
673
+ * {@link SCROLL_STASH_MAX_CHARS} is the real memory bound.
674
+ */
675
+ const SCROLL_STASH_MAX_ENTRIES = 2e3;
676
+ /** Cap on stashed HTML characters (aligned with the pipeline input cap). */
677
+ const SCROLL_STASH_MAX_CHARS = 2e6;
678
+ /**
679
+ * Cap on ONE stashed element's HTML. A virtualized row is a few KB; an
680
+ * element tens of KB big is a CONTAINER (measured on linux.do: six nested
681
+ * wrappers each embed the whole post stream — main 164KB ⊃
682
+ * main-outlet-wrapper 156KB ⊃ … — plus data-preloaded 300KB and
683
+ * svg-sprites 277KB) or a data island — never a row the merge could
684
+ * rehydrate. Such keys are remembered and never serialized again.
685
+ */
686
+ const SCROLL_STASH_MAX_ENTRY_CHARS = 64e3;
687
+ /**
688
+ * Tail of the fetch budget the walk never spends, keeping room for the
689
+ * content read and the synchronous denoise decode.
690
+ */
691
+ const SCROLL_FINISH_RESERVE_MS = 4e3;
692
+ /** The viewport share an inner container's scrollable distance must reach. */
693
+ const SCROLL_CONTAINER_MIN_RANGE_RATIO = .5;
694
+ /**
695
+ * The in-page walk. Evaluated BY SOURCE (`fn.toString()` composed into an
696
+ * expression by {@link deepCaptureScript}), so it must stay self-contained:
697
+ * every helper is nested, every knob arrives in `p`, and the one shared
698
+ * routine — the scripting strip — arrives as an argument. Typed here so the
699
+ * compiler checks what used to be an unchecked string.
700
+ *
701
+ * @param p - the walk's knobs.
702
+ * @param strip - {@link stripScriptingElementsIn}, applied to the merged clone.
703
+ * @returns the walk's stats, plus the merged HTML when anything was restored.
704
+ */
705
+ async function deepCaptureInPage(p, strip) {
706
+ const started = Date.now();
707
+ const root = document.documentElement;
708
+ const body = document.body;
709
+ if (root === null || body === null) return {
710
+ steps: 0,
711
+ stashed: 0,
712
+ rehydrated: 0,
713
+ skipped: "not-scrollable"
714
+ };
715
+ if (p.budgetMs <= 0) return {
716
+ steps: 0,
717
+ stashed: 0,
718
+ rehydrated: 0,
719
+ skipped: "no-budget"
720
+ };
721
+ const KEYED = "[id], [data-block-id]";
722
+ const SVG_NS = "http://www.w3.org/2000/svg";
723
+ const sleep = (ms) => new Promise((resolve) => {
724
+ setTimeout(resolve, ms);
725
+ });
726
+ function findScrollContainer(host) {
727
+ const minRange = Math.max(p.containerMinRangePx, window.innerHeight * p.containerMinRatio);
728
+ let best = null;
729
+ let bestRange = 0;
730
+ for (const el of Array.from(host.querySelectorAll("*"))) {
731
+ const range = el.scrollHeight - el.clientHeight;
732
+ if (range < minRange || range <= bestRange) continue;
733
+ const overflowY = getComputedStyle(el).overflowY;
734
+ if (overflowY === "auto" || overflowY === "scroll") {
735
+ best = el;
736
+ bestRange = range;
737
+ }
738
+ }
739
+ return best;
740
+ }
741
+ const documentScrolls = root.scrollHeight > window.innerHeight + 4;
742
+ const scroller = documentScrolls ? null : findScrollContainer(body);
743
+ if (!documentScrolls && scroller === null) return {
744
+ steps: 0,
745
+ stashed: 0,
746
+ rehydrated: 0,
747
+ skipped: "not-scrollable"
748
+ };
749
+ const viewportH = () => scroller === null ? window.innerHeight : scroller.clientHeight;
750
+ const scrollH = () => scroller === null ? root.scrollHeight : scroller.scrollHeight;
751
+ const curY = () => scroller === null ? window.scrollY : scroller.scrollTop;
752
+ const atBottom = () => curY() + viewportH() >= scrollH() - 2;
753
+ function scrollToY(y) {
754
+ if (scroller === null) window.scrollTo({
755
+ top: y,
756
+ behavior: "instant"
757
+ });
758
+ else scroller.scrollTop = y;
759
+ }
760
+ function smoothScrollBy(dy) {
761
+ return new Promise((resolve) => {
762
+ const fromY = curY();
763
+ const t0 = Date.now();
764
+ const tick = () => {
765
+ const progress = (Date.now() - t0) / p.stepMotionMs;
766
+ if (progress >= 1 || curY() !== fromY && atBottom()) {
767
+ scrollToY(fromY + dy);
768
+ resolve();
769
+ return;
770
+ }
771
+ scrollToY(fromY + Math.round(dy * progress));
772
+ setTimeout(tick, 16);
773
+ };
774
+ tick();
775
+ });
776
+ }
777
+ function keyOf(el) {
778
+ if (el.id !== "") return el.id;
779
+ const block = el.getAttribute("data-block-id");
780
+ return block !== null && block !== "" ? `data-block-id:${block}` : null;
781
+ }
782
+ function siblingKey(el, dir) {
783
+ for (let sib = el[dir]; sib !== null; sib = sib[dir]) {
784
+ const key = keyOf(sib);
785
+ if (key !== null) return key;
786
+ }
787
+ return null;
788
+ }
789
+ const INERT_TAGS = /* @__PURE__ */ new Set([
790
+ "SCRIPT",
791
+ "STYLE",
792
+ "LINK",
793
+ "META",
794
+ "NOSCRIPT",
795
+ "IFRAME",
796
+ "TEMPLATE"
797
+ ]);
798
+ let reactions = 0;
799
+ const counts = (nodes) => {
800
+ for (const node of Array.from(nodes)) if (node.nodeType === 1 && !INERT_TAGS.has(node.nodeName)) return true;
801
+ return false;
802
+ };
803
+ const tally = (records) => {
804
+ for (const record of records) if (counts(record.addedNodes) || counts(record.removedNodes)) reactions++;
805
+ };
806
+ const observer = new MutationObserver(tally);
807
+ observer.observe(body, {
808
+ childList: true,
809
+ subtree: true
810
+ });
811
+ const reacted = () => {
812
+ tally(observer.takeRecords());
813
+ return reactions > 0;
814
+ };
815
+ const stashDoc = document.implementation.createHTMLDocument("");
816
+ const stash = /* @__PURE__ */ new Map();
817
+ const oversized = /* @__PURE__ */ new Set();
818
+ let stashChars = 0;
819
+ const host = body;
820
+ function collect() {
821
+ for (const el of Array.from(host.querySelectorAll(KEYED))) {
822
+ const key = keyOf(el);
823
+ if (key === null || oversized.has(key) || el.namespaceURI === SVG_NS) continue;
824
+ const els = el.childElementCount;
825
+ const text = (el.textContent ?? "").length;
826
+ if (els === 0 && text === 0) continue;
827
+ const prev = stash.get(key);
828
+ if (prev !== void 0) {
829
+ prev.prev = siblingKey(el, "previousElementSibling") ?? prev.prev;
830
+ prev.next = siblingKey(el, "nextElementSibling") ?? prev.next;
831
+ if (els <= prev.els && text <= prev.text) continue;
832
+ } else if (stash.size >= p.stashMaxEntries) continue;
833
+ if (text > p.stashMaxEntryChars) {
834
+ oversized.add(key);
835
+ continue;
836
+ }
837
+ const len = el.outerHTML.length;
838
+ if (len > p.stashMaxEntryChars) {
839
+ oversized.add(key);
840
+ continue;
841
+ }
842
+ const freed = prev === void 0 ? 0 : prev.len;
843
+ if (stashChars - freed + len > p.stashMaxChars) continue;
844
+ stashChars += len - freed;
845
+ stash.set(key, {
846
+ node: stashDoc.importNode(el, true),
847
+ els,
848
+ text,
849
+ len,
850
+ prev: prev?.prev ?? siblingKey(el, "previousElementSibling"),
851
+ next: prev?.next ?? siblingKey(el, "nextElementSibling"),
852
+ inScroller: scroller !== null && scroller.contains(el)
853
+ });
854
+ }
855
+ }
856
+ collect();
857
+ if (curY() > 0) {
858
+ scrollToY(0);
859
+ await sleep(p.stepSettleMs);
860
+ collect();
861
+ }
862
+ tally(observer.takeRecords());
863
+ reactions = 0;
864
+ let steps = 0;
865
+ let lastHeight = scrollH();
866
+ let lastChangeAt = Date.now();
867
+ let careful = false;
868
+ let jumpedFrom = null;
869
+ const noteHeight = () => {
870
+ const height = scrollH();
871
+ if (height !== lastHeight) {
872
+ lastHeight = height;
873
+ lastChangeAt = Date.now();
874
+ }
875
+ };
876
+ try {
877
+ while (steps < p.maxSteps && Date.now() - started < p.budgetMs) {
878
+ if (jumpedFrom !== null && reacted()) {
879
+ scrollToY(jumpedFrom);
880
+ jumpedFrom = null;
881
+ careful = true;
882
+ await sleep(p.stepSettleMs);
883
+ collect();
884
+ noteHeight();
885
+ continue;
886
+ }
887
+ if (atBottom()) {
888
+ await sleep(p.bottomPollMs);
889
+ collect();
890
+ const before = lastHeight;
891
+ noteHeight();
892
+ if (lastHeight === before && Date.now() - lastChangeAt >= p.bottomPatienceMs) break;
893
+ continue;
894
+ }
895
+ if (!careful && jumpedFrom === null) {
896
+ if (reacted()) careful = true;
897
+ else if (steps >= p.inertSteps) {
898
+ jumpedFrom = curY();
899
+ steps++;
900
+ scrollToY(scrollH());
901
+ lastChangeAt = Date.now();
902
+ await sleep(p.stepSettleMs);
903
+ collect();
904
+ noteHeight();
905
+ continue;
906
+ }
907
+ }
908
+ steps++;
909
+ await smoothScrollBy(Math.max(1, Math.round(viewportH() * p.stepRatio)));
910
+ collect();
911
+ await sleep(p.stepSettleMs);
912
+ collect();
913
+ noteHeight();
914
+ }
915
+ } finally {
916
+ observer.disconnect();
917
+ }
918
+ const reactive = reacted();
919
+ const finalHeight = scrollH();
920
+ const fresh = document.cloneNode(true);
921
+ const present = /* @__PURE__ */ new Map();
922
+ const index = (node) => {
923
+ const key = keyOf(node);
924
+ if (key !== null && !present.has(key)) present.set(key, node);
925
+ for (const el of Array.from(node.querySelectorAll(KEYED))) {
926
+ const inner = keyOf(el);
927
+ if (inner !== null && !present.has(inner)) present.set(inner, el);
928
+ }
929
+ };
930
+ if (fresh.documentElement !== null) index(fresh.documentElement);
931
+ let spliced = 0;
932
+ if (scroller !== null) {
933
+ let pending = Array.from(stash.keys()).filter((key) => stash.get(key).inScroller && !present.has(key));
934
+ let progressed = true;
935
+ while (pending.length > 0 && progressed) {
936
+ progressed = false;
937
+ const unresolved = [];
938
+ for (const key of pending) {
939
+ if (present.has(key)) {
940
+ progressed = true;
941
+ continue;
942
+ }
943
+ const snap = stash.get(key);
944
+ const after = snap.prev === null ? void 0 : present.get(snap.prev);
945
+ const before = snap.next === null ? void 0 : present.get(snap.next);
946
+ if (after === void 0 && before === void 0) {
947
+ unresolved.push(key);
948
+ continue;
949
+ }
950
+ const node = fresh.importNode(snap.node, true);
951
+ if (after !== void 0) after.after(node);
952
+ else before.before(node);
953
+ index(node);
954
+ spliced++;
955
+ progressed = true;
956
+ }
957
+ pending = unresolved;
958
+ }
959
+ }
960
+ const degraded = [];
961
+ for (const el of Array.from(fresh.querySelectorAll(KEYED))) {
962
+ const key = keyOf(el);
963
+ const snap = key === null ? void 0 : stash.get(key);
964
+ if (snap === void 0 || el.namespaceURI === SVG_NS) continue;
965
+ if (el.childElementCount < snap.els || snap.text - (el.textContent ?? "").length > p.textDegradeTolerance) degraded.push({
966
+ el,
967
+ snap
968
+ });
969
+ }
970
+ const containers = /* @__PURE__ */ new Set();
971
+ for (const { el } of degraded) for (let up = el.parentElement; up !== null && !containers.has(up); up = up.parentElement) containers.add(up);
972
+ let rehydrated = 0;
973
+ for (const { el, snap } of degraded) {
974
+ if (containers.has(el)) continue;
975
+ el.replaceWith(fresh.importNode(snap.node, true));
976
+ rehydrated++;
977
+ }
978
+ const stats = {
979
+ steps,
980
+ stashed: stash.size,
981
+ rehydrated,
982
+ spliced,
983
+ reactive,
984
+ elapsedMs: Date.now() - started,
985
+ height: finalHeight
986
+ };
987
+ if ((rehydrated > 0 || spliced > 0) && fresh.documentElement !== null) {
988
+ strip(fresh);
989
+ stats.html = (fresh.doctype === null ? "" : new XMLSerializer().serializeToString(fresh.doctype)) + fresh.documentElement.outerHTML;
990
+ }
991
+ return stats;
992
+ }
993
+ /**
994
+ * The walk's knobs for one fetch: the module constants plus its budget.
995
+ *
996
+ * @param budgetMs - wall-clock budget the walk may spend inside the page.
997
+ */
998
+ function deepCaptureParams(budgetMs) {
999
+ return {
1000
+ budgetMs: Math.max(0, Math.round(budgetMs)),
1001
+ stepRatio: SCROLL_STEP_RATIO,
1002
+ stepMotionMs: 300,
1003
+ stepSettleMs: 300,
1004
+ bottomPollMs: 450,
1005
+ bottomPatienceMs: SCROLL_BOTTOM_PATIENCE_MS,
1006
+ inertSteps: 2,
1007
+ maxSteps: 50,
1008
+ stashMaxEntries: SCROLL_STASH_MAX_ENTRIES,
1009
+ stashMaxChars: SCROLL_STASH_MAX_CHARS,
1010
+ stashMaxEntryChars: SCROLL_STASH_MAX_ENTRY_CHARS,
1011
+ containerMinRangePx: 300,
1012
+ containerMinRatio: SCROLL_CONTAINER_MIN_RANGE_RATIO,
1013
+ textDegradeTolerance: 16
1014
+ };
1015
+ }
1016
+ /**
1017
+ * Compose the evaluate expression: the in-page walk applied to `params` and
1018
+ * the shared scripting strip, both by source.
1019
+ *
1020
+ * @param params - the walk's knobs (see {@link deepCaptureParams}).
1021
+ * @returns the expression string to pass to `page.evaluate`.
1022
+ */
1023
+ function deepCaptureExpression(params) {
1024
+ return `/* dsh-deep-capture */ (${deepCaptureInPage.toString()})(${JSON.stringify(params)}, ${stripScriptingElementsIn.toString()})`;
1025
+ }
1026
+ /** {@link deepCaptureExpression} over {@link deepCaptureParams} — one fetch's walk. */
1027
+ function deepCaptureScript(budgetMs) {
1028
+ return deepCaptureExpression(deepCaptureParams(budgetMs));
1029
+ }
1030
+ /**
1031
+ * Run the deep-capture walk in the page, degrading silently to the legacy
1032
+ * single-instant capture on any failure: a page without `evaluate`, a budget
1033
+ * too small to act, a rejected evaluation (the fetch deadline closes the tab
1034
+ * mid-walk), or a result that is not the stats object all just skip it. A
1035
+ * walk can only add content, never fail a fetch.
1036
+ *
1037
+ * @param page - the fetch's tab.
1038
+ * @param budgetMs - wall-clock budget for the in-page walk; budgets under
1039
+ * {@link SCROLL_MIN_BUDGET_MS} skip it.
1040
+ * @returns the walk's stats (with the merged `html` when it restored
1041
+ * anything), or undefined when the walk did not run.
1042
+ */
1043
+ async function deepCaptureWalk(page, budgetMs) {
1044
+ if (typeof page.evaluate !== "function") return void 0;
1045
+ if (budgetMs < 500) return void 0;
1046
+ try {
1047
+ const result = await page.evaluate(deepCaptureScript(budgetMs));
1048
+ if (result === null || typeof result !== "object") return void 0;
1049
+ const raw = result;
1050
+ if (typeof raw.steps !== "number" || typeof raw.stashed !== "number" || typeof raw.rehydrated !== "number") return;
1051
+ return {
1052
+ steps: raw.steps,
1053
+ stashed: raw.stashed,
1054
+ rehydrated: raw.rehydrated,
1055
+ ...typeof raw.spliced === "number" ? { spliced: raw.spliced } : {},
1056
+ ...typeof raw.reactive === "boolean" ? { reactive: raw.reactive } : {},
1057
+ ...typeof raw.elapsedMs === "number" ? { elapsedMs: raw.elapsedMs } : {},
1058
+ ...typeof raw.height === "number" ? { height: raw.height } : {},
1059
+ ...raw.skipped === "not-scrollable" || raw.skipped === "no-budget" ? { skipped: raw.skipped } : {},
1060
+ ...typeof raw.html === "string" ? { html: raw.html } : {}
1061
+ };
1062
+ } catch {
1063
+ return;
1064
+ }
1065
+ }
1066
+ //#endregion
430
1067
  //#region src/session-window.ts
431
1068
  const ownership = /* @__PURE__ */ new WeakMap();
432
1069
  const unavailable = /* @__PURE__ */ new WeakSet();
@@ -1064,8 +1701,14 @@ function installBuiltinSlashResolveCompat() {
1064
1701
  * inline `data:` image payloads are elided to size placeholders (build
1065
1702
  * tools inline images as base64, which would otherwise dominate the body);
1066
1703
  * Mozilla Readability extracts the article (dropping nav bars, sidebars,
1067
- * footers, and ad chrome by scoring link density and text mass); DOMPurify
1068
- * sanitizes whatever HTML remains and forbids the layout tags noise lives in;
1704
+ * footers, and ad chrome by scoring link density and text mass) — its result
1705
+ * must then pass a coverage gate that distrusts tiny or sliver-thin
1706
+ * extractions, which on list-shaped pages (portals, catalogs) is exactly
1707
+ * what the link-density penalty leaves behind; an accepted extraction is
1708
+ * then pruned of trailing link-dense sections — the post-article
1709
+ * recommendation rails and share bars the scorer's vocabularies cannot name
1710
+ * (they arrive as bare divs with no class to match); DOMPurify sanitizes
1711
+ * whatever HTML remains and forbids the layout tags noise lives in;
1069
1712
  * Turndown with the GFM plugin converts to markdown using the same style
1070
1713
  * options and span-safe table rules as the shipped `dsh-tool-web` renderer,
1071
1714
  * so output is consistent with what `web_fetch` produces elsewhere.
@@ -1167,6 +1810,99 @@ function humanSize(bytes) {
1167
1810
  if (bytes < 1048576) return `${(bytes / 1024).toFixed(1)}KB`;
1168
1811
  return `${(bytes / 1048576).toFixed(1)}MB`;
1169
1812
  }
1813
+ /**
1814
+ * Tags whose subtrees are noise when measuring how much of a page an
1815
+ * extraction kept: DOMPurify drops these outright (`KEEP_CONTENT: false`),
1816
+ * and its default policy always removes `script`/`style` — so a text walk
1817
+ * that skips them measures what the sanitized document fallback could
1818
+ * actually return, not text that is already destined for deletion. Measuring
1819
+ * the raw body instead would let a huge `<nav>`/`<footer>` mass shrink an
1820
+ * article's honest coverage share and demote good extractions.
1821
+ */
1822
+ const NOISE_SUBTREE_TAGS = /* @__PURE__ */ new Set([
1823
+ ...FORBID_TAGS,
1824
+ "script",
1825
+ "style"
1826
+ ]);
1827
+ /**
1828
+ * A page must have at least this much visible (post-noise) text before the
1829
+ * article acceptance gate below has an opinion. Sparse pages — logins,
1830
+ * stubs, `login-like` fixtures — have nowhere for content to hide, so
1831
+ * Readability's pick, however short, is still the best reading of the page
1832
+ * and keeps flowing through the article path exactly as before.
1833
+ */
1834
+ const GATE_MIN_BODY_TEXT_CHARS = 1e3;
1835
+ /**
1836
+ * Minimum non-whitespace characters a Readability extraction must carry
1837
+ * before it can be trusted as a page's content. Measured on the 163.com
1838
+ * homepage, the scorer's pick was a 96-character footer licensing block —
1839
+ * real articles and even sparse pages clear two-to-three times this bar,
1840
+ * while link-dense portal pages lose whole news columns to it.
1841
+ */
1842
+ const GATE_MIN_ARTICLE_TEXT_CHARS = 250;
1843
+ /**
1844
+ * Minimum share of a page's visible text an extraction must cover to be
1845
+ * trusted. Measured separation on real pages is wide: portal homepages
1846
+ * (163 0.01, sina 0.03, sohu 0.02) vs article pages (0.28–0.45), so 10%
1847
+ * sits in an empty band with margin on both sides.
1848
+ */
1849
+ const GATE_MIN_ARTICLE_COVERAGE = .1;
1850
+ /**
1851
+ * Count non-whitespace characters under `root`, skipping noise subtrees
1852
+ * ({@link NOISE_SUBTREE_TAGS}). Whitespace is ignored so CJK pages (no
1853
+ * spaces) and latin pages (space-separated) measure comparable densities,
1854
+ * mirroring how the extraction-quality ratios in the docs were computed.
1855
+ *
1856
+ * @param root - the subtree to measure; `null` counts as nothing.
1857
+ * @returns the number of non-whitespace characters in kept subtrees.
1858
+ */
1859
+ function visibleTextLength(root) {
1860
+ if (root === null) return 0;
1861
+ let length = 0;
1862
+ const visit = (node) => {
1863
+ if (node.nodeType === 3) {
1864
+ length += (node.nodeValue ?? "").replace(/\s+/g, "").length;
1865
+ return;
1866
+ }
1867
+ if (node.nodeType !== 1 || NOISE_SUBTREE_TAGS.has(node.nodeName.toLowerCase())) return;
1868
+ for (let child = node.firstChild; child !== null; child = child.nextSibling) visit(child);
1869
+ };
1870
+ visit(root);
1871
+ return length;
1872
+ }
1873
+ /**
1874
+ * Whether a Readability extraction is trustworthy as *the* content of the
1875
+ * page it came from — the acceptance gate that keeps list-shaped pages whole.
1876
+ *
1877
+ * Readability is an article extractor: it scores text mass and penalizes
1878
+ * link density, so on a portal/catalog page — where the content *is* the
1879
+ * links — every news column scores near zero, some sparse low-link block (a
1880
+ * footer licensing notice, a province-switcher popup) wins by default, and
1881
+ * Readability's own 500-char quality check cannot fail the parse because its
1882
+ * retry loop eventually returns the longest of its bad attempts. The plugin
1883
+ * used to accept any non-null result, so the 163.com homepage rendered as a
1884
+ * 162-character licensing fragment while 320 headlines were dropped — a
1885
+ * recall loss far worse than any precision the gate could cost.
1886
+ *
1887
+ * The gate therefore distrusts an extraction that is tiny in absolute terms
1888
+ * or covers a sliver of the page's visible text; the pipeline then falls
1889
+ * back to the sanitized whole document, which DOMPurify keeps clean and
1890
+ * which on such pages retains effectively all list content (measured 0%
1891
+ * text loss on the 163.com homepage). Only text-rich pages are gated
1892
+ * ({@link GATE_MIN_BODY_TEXT_CHARS}); below that, behavior is unchanged.
1893
+ *
1894
+ * @param articleHtml - the `article.content` HTML Readability produced.
1895
+ * @param document - the live document the article was extracted from.
1896
+ * @returns whether the article should flow through the article path.
1897
+ */
1898
+ function articleCoversPage(articleHtml, document) {
1899
+ const bodyText = visibleTextLength(document.body);
1900
+ if (bodyText < GATE_MIN_BODY_TEXT_CHARS) return true;
1901
+ const container = document.createElement("div");
1902
+ container.innerHTML = articleHtml;
1903
+ const articleText = visibleTextLength(container);
1904
+ return articleText >= GATE_MIN_ARTICLE_TEXT_CHARS && articleText / bodyText >= GATE_MIN_ARTICLE_COVERAGE;
1905
+ }
1170
1906
  /** The shared converter: same style options as `dsh-tool-web`'s renderer. */
1171
1907
  const turndown = new TurndownService({
1172
1908
  headingStyle: "atx",
@@ -1215,12 +1951,216 @@ turndown.addRule("tableRowWithoutSpanExpansion", {
1215
1951
  }
1216
1952
  });
1217
1953
  /**
1954
+ * Count non-whitespace characters under `root` that sit inside an anchor,
1955
+ * skipping the same noise subtrees {@link visibleTextLength} skips so the
1956
+ * numerator and denominator of a link-density ratio stay comparable.
1957
+ *
1958
+ * @param root - the subtree to measure; `null` counts as nothing.
1959
+ * @returns the number of non-whitespace characters inside `<a>` descendants.
1960
+ */
1961
+ function linkTextLength(root) {
1962
+ if (root === null) return 0;
1963
+ let length = 0;
1964
+ const visit = (node, insideAnchor) => {
1965
+ if (node.nodeType === 3) {
1966
+ if (insideAnchor) length += (node.nodeValue ?? "").replace(/\s+/g, "").length;
1967
+ return;
1968
+ }
1969
+ if (node.nodeType !== 1 || NOISE_SUBTREE_TAGS.has(node.nodeName.toLowerCase())) return;
1970
+ const anchor = insideAnchor || node.nodeName.toLowerCase() === "a";
1971
+ for (let child = node.firstChild; child !== null; child = child.nextSibling) visit(child, anchor);
1972
+ };
1973
+ visit(root, false);
1974
+ return length;
1975
+ }
1976
+ /**
1977
+ * Minimum non-whitespace characters a block must carry before its link
1978
+ * density is even evaluated. Blocks below this are spacers, empty wrappers,
1979
+ * or one-line sign-offs — too small to be worth a removal decision, so the
1980
+ * trailing walk skips past them instead of stopping at them.
1981
+ */
1982
+ const TRAILING_MIN_TEXT_CHARS = 40;
1983
+ /**
1984
+ * Link-density share above which a trailing block reads as a link farm:
1985
+ * recommendation feeds, related-article rails, tag clouds, share bars.
1986
+ * Measured on the 163.com dy-article badcase the section's pick carried 84
1987
+ * linked characters inside 183 total (0.46) — recommendation cards dilute
1988
+ * their density with author/date lines, so a 0.5 bar would miss them by
1989
+ * construction; an honest closing paragraph with a citation or two measures
1990
+ * 0.1–0.3, leaving 0.4 in the empty band between.
1991
+ */
1992
+ const TRAILING_LINK_DENSITY = .4;
1993
+ /**
1994
+ * A trailing block qualifies as a link farm only when it is link-heavy in
1995
+ * aggregate too: at least {@link TRAILING_LINK_COUNT} anchors or
1996
+ * {@link TRAILING_LINK_TEXT_CHARS} linked characters. Density alone would
1997
+ * let a single long linked sentence (an inline citation) drag a legitimate
1998
+ * paragraph under the knife.
1999
+ */
2000
+ const TRAILING_LINK_COUNT = 3;
2001
+ const TRAILING_LINK_TEXT_CHARS = 60;
2002
+ /**
2003
+ * A trailing block may never hold more than this share of the whole
2004
+ * article's text. The prune targets *appended* widgets; a block that IS most
2005
+ * of the extraction says the scorer picked the widget, which is the coverage
2006
+ * gate's failure mode to handle, not this walk's.
2007
+ */
2008
+ const TRAILING_MAX_ARTICLE_SHARE = .6;
2009
+ /**
2010
+ * A trailing run may never hold more than this share of its container's
2011
+ * text — the aggregate form of the guard above, and the one that keeps
2012
+ * list-shaped extractions whole. A recommendation rail is a *minority*
2013
+ * appended to real content (the 163 rail is 44% of its container); a
2014
+ * portal or feed whose extraction IS a list of link rows (measured on a
2015
+ * Hacker News homepage capture: ~91 trailing rows, every one individually
2016
+ * link-dense at 0.69–0.94) has a tail run holding ~97% of the container —
2017
+ * there the tail IS the content, the walk declines wholesale, and only a
2018
+ * run that is a minority of its container is removed.
2019
+ */
2020
+ const TRAILING_MAX_RUN_SHARE = .6;
2021
+ /**
2022
+ * Link-density band within which a run and the block the walk stopped at
2023
+ * read as the same material. When the content immediately before the run
2024
+ * is itself as link-dense as the run — measured on a Hacker News homepage
2025
+ * capture, the rows after the run's cut measure 0.94 against the run's
2026
+ * 0.91 — the container is a *uniform list* (feed rows, portal headlines)
2027
+ * and the tail is that list's own content, not a widget appended after it;
2028
+ * a widget hangs after genuinely different material (the 163 rail's 0.44
2029
+ * follows body text at 0.00). Only a run denser than what precedes it is
2030
+ * removed.
2031
+ */
2032
+ const TRAILING_UNIFORM_DENSITY_BAND = .25;
2033
+ /**
2034
+ * Prune trailing link-dense sections from an accepted Readability
2035
+ * extraction — the post-article widget class the scorer itself cannot be
2036
+ * taught to drop.
2037
+ *
2038
+ * Why this exists (badcase: `https://www.163.com/dy/article/…`): Chinese
2039
+ * portal article pages append a recommendation feed inside the very wrapper
2040
+ * Readability scores as the article (`#content` holds the body, the
2041
+ * platform disclaimer, then a rail of image/title/author/date cards). Every
2042
+ * existing noise layer misses it: Readability's `unlikelyCandidates` /
2043
+ * `negative` vocabularies know `related` but not `recommend` (nor the
2044
+ * pinyin families `tuijian`/`xiangguan`), and its own cleaning pass draws
2045
+ * the "mostly links" line at per-block density 0.5 — these cards measure
2046
+ * 0.42–0.48 (the title links over title+author+date text), so every card
2047
+ * clears it; in the extracted HTML the rail is a *bare* div — class and id
2048
+ * are gone — so no class-based rule can ever see it again; DOMPurify's
2049
+ * FORBID_TAGS works on tags, and the rail is plain div/p/h3/a; and the
2050
+ * coverage gate cannot help because the noise rides *inside* the
2051
+ * extraction it measures (the badcase passed at 0.103 — the rail's text
2052
+ * was in both numerator and denominator). Modern sites with CSS-module
2053
+ * class names (`css-1b3x7r`) present the same shape with no vocabulary to
2054
+ * match at all, so the only signal that survives all of this is
2055
+ * structural: a *trailing* run of blocks whose text is mostly links.
2056
+ *
2057
+ * For every container in the extraction the walk gathers the maximal
2058
+ * trailing run of qualifying children — link-farm blocks per the constants
2059
+ * above, plus sub-threshold spacers (empty wrapper divs) — and removes the
2060
+ * run only when BOTH structural guards say "appended widget, not content":
2061
+ * the run is a minority of the container's text
2062
+ * ({@link TRAILING_MAX_RUN_SHARE}), and the run is clearly denser than the
2063
+ * block the walk stopped at ({@link TRAILING_UNIFORM_DENSITY_BAND}). The
2064
+ * guards split the two worlds the per-block view cannot: a rail is
2065
+ * one-or-few appended blocks after genuinely different material (the 163
2066
+ * rail's 0.44 density follows body text at 0.00), while a link-list
2067
+ * extraction — feed rows, portal headlines — is *uniform*: its rows before
2068
+ * any cut measure the same density as the rows after it (0.91 vs 0.94 on
2069
+ * the Hacker News capture), so no tail of it can be mistaken for a widget.
2070
+ *
2071
+ * Deliberately NOT applied to the whole-document fallback: that path
2072
+ * exists for list-shaped pages where the links *are* the content (the
2073
+ * 163.com homepage fix), and pruning link-dense regions there would
2074
+ * re-introduce exactly the recall loss the coverage gate exists to
2075
+ * prevent.
2076
+ *
2077
+ * @param container - parsed `article.content`; mutated in place.
2078
+ */
2079
+ function pruneTrailingLinkDenseSections(container) {
2080
+ const articleText = visibleTextLength(container);
2081
+ if (articleText === 0) return;
2082
+ for (const element of [container, ...Array.from(container.querySelectorAll("*"))]) {
2083
+ if (!container.contains(element) && element !== container) continue;
2084
+ removeTrailingLinkDenseRun(element, articleText);
2085
+ }
2086
+ }
2087
+ /**
2088
+ * Remove one container's maximal trailing run of link-dense children when
2089
+ * that run is a minority of the container's text AND clearly different
2090
+ * material from the content that precedes it.
2091
+ *
2092
+ * @param parent - the container whose element children are examined from
2093
+ * the end.
2094
+ * @param articleText - the whole article's text, for the per-block share
2095
+ * guard.
2096
+ */
2097
+ function removeTrailingLinkDenseRun(parent, articleText) {
2098
+ const children = Array.from(parent.children);
2099
+ if (children.length === 0) return;
2100
+ const containerText = visibleTextLength(parent);
2101
+ if (containerText === 0) return;
2102
+ const run = [];
2103
+ let runText = 0;
2104
+ let runLinked = 0;
2105
+ let stopIndex = -1;
2106
+ for (let index = children.length - 1; index >= 0; index--) {
2107
+ const child = children[index];
2108
+ if (child === void 0) break;
2109
+ const text = visibleTextLength(child);
2110
+ if (text < TRAILING_MIN_TEXT_CHARS) {
2111
+ run.push(child);
2112
+ continue;
2113
+ }
2114
+ const linked = linkTextLength(child);
2115
+ const linkCount = child.querySelectorAll("a").length;
2116
+ if (linked / text >= TRAILING_LINK_DENSITY && (linkCount >= TRAILING_LINK_COUNT || linked >= TRAILING_LINK_TEXT_CHARS) && text <= TRAILING_MAX_ARTICLE_SHARE * articleText) {
2117
+ run.push(child);
2118
+ runText += text;
2119
+ runLinked += linked;
2120
+ continue;
2121
+ }
2122
+ stopIndex = index;
2123
+ break;
2124
+ }
2125
+ if (runText === 0 || stopIndex === -1) return;
2126
+ const stopChild = children[stopIndex];
2127
+ if (stopChild === void 0) return;
2128
+ const stopDensity = linkTextLength(stopChild) / Math.max(1, visibleTextLength(stopChild));
2129
+ if (Math.abs(runLinked / runText - stopDensity) <= TRAILING_UNIFORM_DENSITY_BAND) return;
2130
+ if (runText > TRAILING_MAX_RUN_SHARE * containerText) return;
2131
+ for (const child of run) child.remove();
2132
+ }
2133
+ /**
2134
+ * Run {@link pruneTrailingLinkDenseSections} over one extraction with a
2135
+ * misfire guard: a prune that takes a healthy article below the gate's
2136
+ * absolute floor ({@link GATE_MIN_ARTICLE_TEXT_CHARS}) or empties it
2137
+ * entirely says the walk misread the page, and Readability's original pick
2138
+ * is returned untouched. A *sparse* article (under the floor before the
2139
+ * prune, auto-trusted by the gate) keeps its pruned form — its trailing
2140
+ * links were noise all the same.
2141
+ *
2142
+ * @param articleHtml - the `article.content` HTML Readability produced.
2143
+ * @param document - the live document, for parsing the fragment.
2144
+ * @returns the pruned article HTML, or the original on a misfire.
2145
+ */
2146
+ function prunedArticleContent(articleHtml, document) {
2147
+ const container = document.createElement("div");
2148
+ container.innerHTML = articleHtml;
2149
+ const before = visibleTextLength(container);
2150
+ if (before === 0) return articleHtml;
2151
+ pruneTrailingLinkDenseSections(container);
2152
+ const after = visibleTextLength(container);
2153
+ if (after === 0 || before >= GATE_MIN_ARTICLE_TEXT_CHARS && after < GATE_MIN_ARTICLE_TEXT_CHARS) return articleHtml;
2154
+ return container.innerHTML;
2155
+ }
2156
+ /**
1218
2157
  * Convert one rendered HTML document to denoised markdown.
1219
2158
  *
1220
- * Readability failure (non-article pages) degrades to converting the
1221
- * sanitized whole document — layout tags are still forbidden, so the
1222
- * fallback stays cleaner than raw turndown, and a degraded page beats an
1223
- * error for a body the browser already rendered.
2159
+ * Readability failure (non-article pages, or extractions the coverage gate
2160
+ * {@link articleCoversPage} distrusts) degrades to converting the sanitized
2161
+ * whole document — layout tags are still forbidden, so the fallback stays
2162
+ * cleaner than raw turndown, and a degraded page beats an error for a body
2163
+ * the browser already rendered.
1224
2164
  *
1225
2165
  * @param html - the rendered page HTML (`page.content()`).
1226
2166
  * @param url - the page URL, used to resolve relative links during parsing.
@@ -1240,8 +2180,8 @@ function htmlToMarkdown(html, url) {
1240
2180
  try {
1241
2181
  const cloned = document.cloneNode(true);
1242
2182
  const article = new Readability(cloned).parse();
1243
- if (article !== null && typeof article.content === "string" && article.content !== "") {
1244
- source = article.content;
2183
+ if (article !== null && typeof article.content === "string" && article.content !== "" && articleCoversPage(article.content, document)) {
2184
+ source = prunedArticleContent(article.content, document);
1245
2185
  title = article.title ?? void 0;
1246
2186
  }
1247
2187
  } catch {}
@@ -1499,6 +2439,31 @@ function existsAsFile(path) {
1499
2439
  * fails with {@link WEB_FETCH_CHALLENGE_CODE} instead of returning the
1500
2440
  * interstitial as content.
1501
2441
  *
2442
+ * Deep capture (viewport-virtualized feeds and app-shell documents): feed-
2443
+ * shaped pages — Discourse topic streams, comment threads, timelines —
2444
+ * render only the items near the scroll position; everything else is an
2445
+ * empty, height-only placeholder that keeps the item's id but none of its
2446
+ * content, so a single-instant capture hands back the skeleton while the
2447
+ * bodies silently vanish (and the denoise pipeline's coverage gate cannot
2448
+ * notice: the page's own visible text is the same windowed fragment as the
2449
+ * extraction). App-shell document editors (Feishu docs) push the same
2450
+ * pathology one level down: the body never scrolls, blocks are keyed by
2451
+ * data-block-id with no id, and off-window blocks are REMOVED behind an
2452
+ * anonymous height spacer — the fetch would return just the statically
2453
+ * rendered first screen. Before serialization the fetch therefore runs one
2454
+ * bounded in-page scroll walk — over the window or, when the document does
2455
+ * not scroll, the page's dominant inner scroll container — stashing what
2456
+ * each rendered window shows, terminating only AT the bottom with the
2457
+ * height stable (that is what pages load-more sentinels; height stability
2458
+ * elsewhere proves nothing), then rehydrating degraded keyed elements from
2459
+ * their stashed snapshots and splicing removed keys back at their document
2460
+ * position so the final DOM holds the union of every window in document
2461
+ * order. The existing pipeline then runs unchanged. Bounded by
2462
+ * `scrollBudgetMs` (default 20s; 0 restores the legacy single-instant
2463
+ * behavior) and by the remaining fetch deadline; any walk failure silently
2464
+ * degrades to the legacy capture (see `src/autoscroll.ts` for the measured
2465
+ * linux.do and Feishu badcases).
2466
+ *
1502
2467
  * @module dsh-web-fetch-playwright/provider
1503
2468
  */
1504
2469
  /** Stable id this provider registers under (the bundle patch pins it). */
@@ -1856,8 +2821,14 @@ var PlaywrightFetchProvider = class {
1856
2821
  content: finalResponse !== null ? await finalResponse.text() : await page.content()
1857
2822
  });
1858
2823
  await page.waitForLoadState("networkidle", { timeout: Math.min(SETTLE_MS, deadline.remainingMs()) }).catch(() => {});
1859
- await stripScriptingElements(page);
1860
- const html = await page.content();
2824
+ const scrollBudgetMs = Math.min(effectiveScrollBudgetMs(config), deadline.remainingMs() - SCROLL_FINISH_RESERVE_MS);
2825
+ const captured = scrollBudgetMs > 0 ? await deepCaptureWalk(page, scrollBudgetMs) : void 0;
2826
+ let html;
2827
+ if (captured?.html !== void 0) html = captured.html;
2828
+ else {
2829
+ await stripScriptingElements(page);
2830
+ html = await page.content();
2831
+ }
1861
2832
  if (!config.denoise) return capResult(finalUrl, statusCode, {
1862
2833
  kind: "html",
1863
2834
  content: html
@@ -2170,66 +3141,12 @@ async function probeStillOnChallenge(page) {
2170
3141
  }
2171
3142
  }
2172
3143
  /**
2173
- * In-page script that removes non-content scripting subtrees from the live
2174
- * document before `page.content()` reads it as an HTML string.
2175
- *
2176
- * WHAT goes, and why each removal is output-neutral versus the old pipeline:
2177
- *
2178
- * - `script` — three consumers ever saw script text before, and none loses
2179
- * it here: Readability's `parse()` deletes script/noscript nodes BEFORE
2180
- * extraction and scoring (so script text never reached the article or
2181
- * influenced the scorer); DOMPurify's default allowlist drops script; the
2182
- * tool layer's renderer removes SCRIPT wholesale on the raw path. What the
2183
- * removal buys is the reason this exists: a page's runtime JS may rewrite
2184
- * a data script's text with a literal `<\/script>` embedded — measured on
2185
- * techcommunity.microsoft.com (Khoros/Aurora), whose client writes its
2186
- * Apollo cache back into `script#__NEXT_DATA__` via `JSON.stringify`,
2187
- * turning the wire's 712 `\u003c` escapes back into raw `<` — and
2188
- * browsers serialize script text verbatim, so the flattened string ends
2189
- * that script at the FIRST embedded end tag and re-parses the remaining
2190
- * ~270KB of JSON as markup. The phantom elements that survive (unknown
2191
- * tags like `<id>`, anchors full of JSON text) are exactly what
2192
- * Readability's scorer reads as article mass, so the extracted "article"
2193
- * is CSS and i18n strings while the real body — still present in the DOM —
2194
- * loses. No string-level patch can undo this after serialization (the
2195
- * string is genuinely ambiguous); only the browser's own parse holds the
2196
- * document's true structure.
2197
- * - `script[type="application/ld+json"]` — KEPT, but defused: Readability
2198
- * extracts metadata (title/byline/excerpt — e.g. the site-suffix-free
2199
- * "Remote Desktop Protocol" instead of "… - Wikipedia") from JSON-LD
2200
- * scripts before removing them, so deleting them would regress the title
2201
- * on every JSON-LD-carrying page. Rewriting `</` as `<\/` keeps the text
2202
- * valid JSON (`\/` is a legal escape, so the re-parsed text parses to the
2203
- * same object) while making the serialization harmless: the tokenizer
2204
- * only terminates on a literal `<\/script`.
2205
- * - `noscript` — KEPT: Readability's `_unwrapNoscriptImages` salvages the
2206
- * real image out of lazy-load `<noscript><img …>` wrappers before
2207
- * `_removeScripts` deletes the noscript; pre-stripping it would silently
2208
- * drop those images from the markdown.
2209
- * - `template` — removed: its children parse into the inert `.content`
2210
- * fragment (invisible to Readability's traversal), DOMPurify's
2211
- * FORBID_TAGS drops it, and the tool layer's renderer removes it too.
2212
- * - `style` stays — its text cannot terminate a re-parse except at its own
2213
- * end tag, and the pipeline already drops it.
2214
- */
2215
- const STRIP_SCRIPTING_ELEMENTS = `(() => {
2216
- for (const el of Array.from(document.querySelectorAll('script, template'))) {
2217
- if (el.tagName === 'SCRIPT' && (el.getAttribute('type') || '').trim().toLowerCase() === 'application/ld+json') {
2218
- // '<\\/' is still valid JSON, so the re-parsed textContent parses to
2219
- // the same object — while the serialized form can no longer close the
2220
- // script element early.
2221
- el.textContent = el.textContent.replace(/<\\//g, '<\\\\/')
2222
- continue
2223
- }
2224
- el.remove()
2225
- }
2226
- })()`;
2227
- /**
2228
3144
  * Remove non-content scripting subtrees from the live document before
2229
- * serialization. Best-effort, mirroring {@link probeStillOnChallenge}: a
2230
- * backend without `evaluate` (fakes, minimal engines) simply keeps the
2231
- * document as-is — the fetch never fails here, and the pipeline degrades to
2232
- * whatever the serialized string itself says.
3145
+ * serialization (see `stripScriptingElementsIn` in `src/strip.ts` for exactly
3146
+ * what goes and why). Best-effort, mirroring {@link probeStillOnChallenge}: a backend
3147
+ * without `evaluate` (fakes, minimal engines) simply keeps the document
3148
+ * as-is — the fetch never fails here, and the pipeline degrades to whatever
3149
+ * the serialized string itself says.
2233
3150
  */
2234
3151
  async function stripScriptingElements(page) {
2235
3152
  if (typeof page.evaluate !== "function") return;
@@ -2366,4 +3283,4 @@ function apply(ctx, config) {
2366
3283
  ctx.web.registerFetchProvider(provider);
2367
3284
  }
2368
3285
  //#endregion
2369
- export { CHALLENGE_DOM_PROBE, CHALLENGE_FINISH_RESERVE_MS, CHALLENGE_POLL_INTERVAL_MS, CHALLENGE_TITLE_RE, CdpConnectionPool, Config, DEFAULT_CDP_ENDPOINT, DEFAULT_CHALLENGE_RETRIES, DEFAULT_CHALLENGE_WAIT_MS, PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider, WEB_FETCH_CHALLENGE_CODE, WEB_FETCH_PLAYWRIGHT_SETTINGS_NAMESPACE, apply, classifyChallengeHtml, classifyChallengeResponse, effectiveChallengeRetries, effectiveChallengeWaitMs, effectiveContextMode, htmlToMarkdown, inject, isChallengeCompatibleResponse, name, normalizeCdpEndpoint, resolveConfig, snapshotsOf };
3286
+ export { CHALLENGE_DOM_PROBE, CHALLENGE_FINISH_RESERVE_MS, CHALLENGE_POLL_INTERVAL_MS, CHALLENGE_TITLE_RE, CdpConnectionPool, Config, DEFAULT_CDP_ENDPOINT, DEFAULT_CHALLENGE_RETRIES, DEFAULT_CHALLENGE_WAIT_MS, DEFAULT_SCROLL_BUDGET_MS, MAX_SCROLL_BUDGET_MS, PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider, WEB_FETCH_CHALLENGE_CODE, WEB_FETCH_PLAYWRIGHT_SETTINGS_NAMESPACE, apply, classifyChallengeHtml, classifyChallengeResponse, deepCaptureWalk, effectiveChallengeRetries, effectiveChallengeWaitMs, effectiveContextMode, effectiveScrollBudgetMs, htmlToMarkdown, inject, isChallengeCompatibleResponse, name, normalizeCdpEndpoint, resolveConfig, snapshotsOf };