dsh-web-fetch-playwright 0.2.16 → 0.2.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -3
- package/README.zh-CN.md +30 -3
- package/lib/client.js +42 -17
- package/lib/index.js +866 -71
- package/lib/types/autoscroll.d.ts +277 -0
- package/lib/types/client/controller.d.ts +7 -0
- package/lib/types/client/locales.d.ts +1 -1
- package/lib/types/config.d.ts +38 -3
- package/lib/types/index.d.ts +3 -1
- package/lib/types/markdown.d.ts +8 -5
- package/lib/types/provider.d.ts +25 -0
- package/lib/types/strip.d.ts +59 -0
- package/package.json +1 -1
- package/src/autoscroll.ts +653 -0
- package/src/client/card.tsx +13 -0
- package/src/client/controller.ts +9 -0
- package/src/client/locales.ts +7 -0
- package/src/config.ts +46 -3
- package/src/index.ts +5 -0
- package/src/markdown.ts +252 -6
- package/src/provider.ts +57 -68
- package/src/strip.ts +72 -0
package/lib/index.js
CHANGED
|
@@ -9,6 +9,17 @@ import { fileURLToPath, pathToFileURL } from "node:url";
|
|
|
9
9
|
import { accessSync, constants, readFileSync, realpathSync, statSync } from "node:fs";
|
|
10
10
|
import { delimiter, dirname, join } from "node:path";
|
|
11
11
|
//#region src/config.ts
|
|
12
|
+
/**
|
|
13
|
+
* Default bounded budget (ms) for the deep-capture scroll walk
|
|
14
|
+
* (`src/autoscroll.ts`). Sized so long topics stay fully covered at the
|
|
15
|
+
* careful cadence (300ms motion + 300ms rest per ~0.7-viewport step, plus
|
|
16
|
+
* the bottom patience window): 20s walks ~25kpx with pagination, covering
|
|
17
|
+
* the measured linux.do badcase (~18kpx, 51 replies) with headroom.
|
|
18
|
+
* Scroll-inert pages take the walk's fast path and spend a few seconds.
|
|
19
|
+
*/
|
|
20
|
+
const DEFAULT_SCROLL_BUDGET_MS = 2e4;
|
|
21
|
+
/** Ceiling the schema accepts for `scrollBudgetMs`. */
|
|
22
|
+
const MAX_SCROLL_BUDGET_MS = 3e4;
|
|
12
23
|
/** Default CDP endpoint when the settings section leaves it blank. */
|
|
13
24
|
const DEFAULT_CDP_ENDPOINT = "127.0.0.1:9222";
|
|
14
25
|
/**
|
|
@@ -31,7 +42,8 @@ const Config = z.object({
|
|
|
31
42
|
denoise: z.boolean().default(true).volatile(),
|
|
32
43
|
maxConcurrency: z.number().step(1).min(1).max(200).volatile(),
|
|
33
44
|
challengeWaitMs: z.number().step(100).min(0).max(MAX_CHALLENGE_WAIT_MS).default(DEFAULT_CHALLENGE_WAIT_MS).volatile(),
|
|
34
|
-
challengeRetries: z.number().step(1).min(0).max(3).default(1).volatile()
|
|
45
|
+
challengeRetries: z.number().step(1).min(0).max(3).default(1).volatile(),
|
|
46
|
+
scrollBudgetMs: z.number().step(100).min(0).max(MAX_SCROLL_BUDGET_MS).default(DEFAULT_SCROLL_BUDGET_MS).volatile()
|
|
35
47
|
});
|
|
36
48
|
/**
|
|
37
49
|
* Read the live references `apply` holds into one plain input for
|
|
@@ -51,7 +63,8 @@ function snapshotsOf(config) {
|
|
|
51
63
|
denoise: config.denoise.get(),
|
|
52
64
|
maxConcurrency: config.maxConcurrency.get(),
|
|
53
65
|
challengeWaitMs: config.challengeWaitMs.get(),
|
|
54
|
-
challengeRetries: config.challengeRetries.get()
|
|
66
|
+
challengeRetries: config.challengeRetries.get(),
|
|
67
|
+
scrollBudgetMs: config.scrollBudgetMs.get()
|
|
55
68
|
};
|
|
56
69
|
}
|
|
57
70
|
/**
|
|
@@ -75,7 +88,8 @@ function resolveConfig(config) {
|
|
|
75
88
|
denoise: resolved.denoise.get(),
|
|
76
89
|
maxConcurrency: resolved.maxConcurrency.get(),
|
|
77
90
|
challengeWaitMs: resolved.challengeWaitMs.get(),
|
|
78
|
-
challengeRetries: resolved.challengeRetries.get()
|
|
91
|
+
challengeRetries: resolved.challengeRetries.get(),
|
|
92
|
+
scrollBudgetMs: resolved.scrollBudgetMs.get()
|
|
79
93
|
};
|
|
80
94
|
}
|
|
81
95
|
/**
|
|
@@ -113,6 +127,17 @@ function effectiveChallengeRetries(config) {
|
|
|
113
127
|
return 1;
|
|
114
128
|
}
|
|
115
129
|
/**
|
|
130
|
+
* The scroll-walk budget a fetch actually runs with: an explicit setting
|
|
131
|
+
* wins, else the schema default. `0` means the deep-capture walk is off.
|
|
132
|
+
*
|
|
133
|
+
* @param config - the resolved settings section (or any partial of it).
|
|
134
|
+
* @returns the bounded walk budget in milliseconds.
|
|
135
|
+
*/
|
|
136
|
+
function effectiveScrollBudgetMs(config) {
|
|
137
|
+
if (typeof config.scrollBudgetMs === "number") return config.scrollBudgetMs;
|
|
138
|
+
return DEFAULT_SCROLL_BUDGET_MS;
|
|
139
|
+
}
|
|
140
|
+
/**
|
|
116
141
|
* The context mode a fetch actually runs with: profile (a tab in the remote
|
|
117
142
|
* browser's real profile) requires the CDP backend — a local launch has no
|
|
118
143
|
* meaningful shared profile — and the share option (which defaults on, so an
|
|
@@ -452,6 +477,593 @@ const CHALLENGE_DOM_PROBE = `(() => {
|
|
|
452
477
|
return false
|
|
453
478
|
})()`;
|
|
454
479
|
//#endregion
|
|
480
|
+
//#region src/strip.ts
|
|
481
|
+
/**
|
|
482
|
+
* Serialization safety for documents about to be flattened into an HTML
|
|
483
|
+
* string and re-parsed by jsdom: the one in-page cleanup both serialization
|
|
484
|
+
* paths share — the provider's live-document read and the deep-capture
|
|
485
|
+
* walk's merged clone.
|
|
486
|
+
*
|
|
487
|
+
* @module dsh-web-fetch-playwright/strip
|
|
488
|
+
*/
|
|
489
|
+
/**
|
|
490
|
+
* In-page function that removes non-content scripting subtrees from a
|
|
491
|
+
* document before it is flattened into an HTML string — the live document
|
|
492
|
+
* before `page.content()`, or the deep-capture walk's merged clone before it
|
|
493
|
+
* serializes it itself (`src/autoscroll.ts`).
|
|
494
|
+
*
|
|
495
|
+
* It is evaluated BY SOURCE (`fn.toString()`), so it must stay
|
|
496
|
+
* self-contained: no imports, no module-level references.
|
|
497
|
+
*
|
|
498
|
+
* WHAT goes, and why each removal is output-neutral versus the old pipeline:
|
|
499
|
+
*
|
|
500
|
+
* - `script` — three consumers ever saw script text before, and none loses
|
|
501
|
+
* it here: Readability's `parse()` deletes script/noscript nodes BEFORE
|
|
502
|
+
* extraction and scoring (so script text never reached the article or
|
|
503
|
+
* influenced the scorer); DOMPurify's default allowlist drops script; the
|
|
504
|
+
* tool layer's renderer removes SCRIPT wholesale on the raw path. What the
|
|
505
|
+
* removal buys is the reason this exists: a page's runtime JS may rewrite
|
|
506
|
+
* a data script's text with a literal `<\/script>` embedded — measured on
|
|
507
|
+
* techcommunity.microsoft.com (Khoros/Aurora), whose client writes its
|
|
508
|
+
* Apollo cache back into `script#__NEXT_DATA__` via `JSON.stringify`,
|
|
509
|
+
* turning the wire's 712 `\u003c` escapes back into raw `<` — and
|
|
510
|
+
* browsers serialize script text verbatim, so the flattened string ends
|
|
511
|
+
* that script at the FIRST embedded end tag and re-parses the remaining
|
|
512
|
+
* ~270KB of JSON as markup. The phantom elements that survive (unknown
|
|
513
|
+
* tags like `<id>`, anchors full of JSON text) are exactly what
|
|
514
|
+
* Readability's scorer reads as article mass, so the extracted "article"
|
|
515
|
+
* is CSS and i18n strings while the real body — still present in the DOM —
|
|
516
|
+
* loses. No string-level patch can undo this after serialization (the
|
|
517
|
+
* string is genuinely ambiguous); only the browser's own parse holds the
|
|
518
|
+
* document's true structure.
|
|
519
|
+
* - `script[type="application/ld+json"]` — KEPT, but defused: Readability
|
|
520
|
+
* extracts metadata (title/byline/excerpt — e.g. the site-suffix-free
|
|
521
|
+
* "Remote Desktop Protocol" instead of "… - Wikipedia") from JSON-LD
|
|
522
|
+
* scripts before removing them, so deleting them would regress the title
|
|
523
|
+
* on every JSON-LD-carrying page. Rewriting `</` as `<\/` keeps the text
|
|
524
|
+
* valid JSON (`\/` is a legal escape, so the re-parsed text parses to the
|
|
525
|
+
* same object) while making the serialization harmless: the tokenizer
|
|
526
|
+
* only terminates on a literal `<\/script`.
|
|
527
|
+
* - `noscript` — KEPT: Readability's `_unwrapNoscriptImages` salvages the
|
|
528
|
+
* real image out of lazy-load `<noscript><img …>` wrappers before
|
|
529
|
+
* `_removeScripts` deletes the noscript; pre-stripping it would silently
|
|
530
|
+
* drop those images from the markdown.
|
|
531
|
+
* - `template` — removed: its children parse into the inert `.content`
|
|
532
|
+
* fragment (invisible to Readability's traversal), DOMPurify's
|
|
533
|
+
* FORBID_TAGS drops it, and the tool layer's renderer removes it too.
|
|
534
|
+
* - `style` stays — its text cannot terminate a re-parse except at its own
|
|
535
|
+
* end tag, and the pipeline already drops it.
|
|
536
|
+
*/
|
|
537
|
+
function stripScriptingElementsIn(doc) {
|
|
538
|
+
for (const el of Array.from(doc.querySelectorAll("script, template"))) {
|
|
539
|
+
if (el.tagName === "SCRIPT" && (el.getAttribute("type") ?? "").trim().toLowerCase() === "application/ld+json") {
|
|
540
|
+
el.textContent = (el.textContent ?? "").replace(/<\//g, "<\\/");
|
|
541
|
+
continue;
|
|
542
|
+
}
|
|
543
|
+
el.remove();
|
|
544
|
+
}
|
|
545
|
+
}
|
|
546
|
+
/** {@link stripScriptingElementsIn} over the live document, as an evaluate expression. */
|
|
547
|
+
const STRIP_SCRIPTING_ELEMENTS = `(${stripScriptingElementsIn.toString()})(document)`;
|
|
548
|
+
//#endregion
|
|
549
|
+
//#region src/autoscroll.ts
|
|
550
|
+
/**
|
|
551
|
+
* Deep capture: a bounded in-page scroll walk that defeats viewport
|
|
552
|
+
* virtualization before the document is serialized.
|
|
553
|
+
*
|
|
554
|
+
* WHY. Long "feed-shaped" pages — Discourse topic streams (linux.do,
|
|
555
|
+
* meta.discourse.org), comment threads, timelines — increasingly render only
|
|
556
|
+
* the posts near the current scroll position. Discourse is the measured
|
|
557
|
+
* badcase: at rest the live DOM holds the posts within roughly one viewport
|
|
558
|
+
* of the top, while every post outside that window is a height-only
|
|
559
|
+
* placeholder that keeps the post's identity but none of its content:
|
|
560
|
+
*
|
|
561
|
+
* ```html
|
|
562
|
+
* <div class="post-stream--cloaked" data-post-number="1" id="post_1"
|
|
563
|
+
* style="height:297.9px !important;…">
|
|
564
|
+
* <h2 class="sr-only">由 Diver-QC 于 2 小时前 发布</h2>
|
|
565
|
+
* </div>
|
|
566
|
+
* ```
|
|
567
|
+
*
|
|
568
|
+
* The placeholder carries the SAME id the rendered `<article id="post_1">`
|
|
569
|
+
* has — every mainstream virtualized list needs some stable identity to
|
|
570
|
+
* recycle its rows. A fetch that renders, never scrolls, and serializes one
|
|
571
|
+
* instant of that DOM returns the skeleton (the screen-reader-only "posted
|
|
572
|
+
* by" headings survive the pipeline as if they were content) while the reply
|
|
573
|
+
* bodies silently vanish. Scrolling to the bottom once does not help either:
|
|
574
|
+
* the window MOVES — measured on the same page, scrolling to the bottom left
|
|
575
|
+
* only posts 31–40 rendered and re-cloaked 1–30. And the denoise pipeline's
|
|
576
|
+
* own coverage gate cannot notice the loss: the page's "visible text" it
|
|
577
|
+
* measures against is the same windowed fragment as the extraction, so the
|
|
578
|
+
* ratio always passes. Only walk depth can see what a resting viewport hides.
|
|
579
|
+
*
|
|
580
|
+
* The SAME pathology, one level deeper, hits app-shell document editors —
|
|
581
|
+
* the second measured badcase is a Feishu wiki page (bocha-ai.feishu.cn).
|
|
582
|
+
* There the body never scrolls at all: html/body are locked at exactly one
|
|
583
|
+
* viewport and the document scrolls inside `div.bear-web-x-container`
|
|
584
|
+
* (measured: documentElement.scrollHeight 720 == innerHeight,
|
|
585
|
+
* window.scrollTo(0, 800) left scrollY at 0, while the container held 4534px
|
|
586
|
+
* against a 656px client). Its blocks are virtualized too, with two twists
|
|
587
|
+
* over Discourse: blocks carry `data-block-id` and NO id, and off-window
|
|
588
|
+
* blocks are REMOVED outright — the whole un-rendered tail is one anonymous
|
|
589
|
+
* `.bear-virtual-renderUnit-placeholder` spacer with an explicit height — so
|
|
590
|
+
* there is neither a window scrollbar to walk nor a degraded-in-place row to
|
|
591
|
+
* swap back. A fetch therefore returns the statically rendered first screen.
|
|
592
|
+
*
|
|
593
|
+
* HOW. Between the settle and the serialization the provider runs one
|
|
594
|
+
* in-page async walk ({@link deepCaptureInPage}, a typed function evaluated
|
|
595
|
+
* by source):
|
|
596
|
+
*
|
|
597
|
+
* 1. Resolve the walk TARGET: the window when the document itself scrolls
|
|
598
|
+
* (the Discourse shape), else the page's dominant inner scroll container
|
|
599
|
+
* (the app-shell shape) — the element with the largest scrollable
|
|
600
|
+
* distance among computed `overflow-y: auto|scroll` elements, gated at
|
|
601
|
+
* half a viewport so dropdown-sized scrollers never trigger a walk. A
|
|
602
|
+
* page with neither is left alone (the legacy capture).
|
|
603
|
+
* 2. STASH every keyed content element — key = id, else a namespaced
|
|
604
|
+
* `data-block-id` — as a detached clone (the RICHEST sighting per key
|
|
605
|
+
* wins), together with its nearest keyed siblings: where it sits in its
|
|
606
|
+
* own list, which is what a removed row is later spliced back next to.
|
|
607
|
+
* 3. WALK down from the top in smooth 0.7-viewport steps (300ms motion +
|
|
608
|
+
* 300ms quiet settle — the rest a feed engine's debounced uncloak needs),
|
|
609
|
+
* collecting after each, until the walk can hold AT THE BOTTOM through a
|
|
610
|
+
* {@link SCROLL_BOTTOM_PATIENCE_MS} window with the height unchanged
|
|
611
|
+
* (growth restarts the clock — that is what pages load-more sentinels),
|
|
612
|
+
* or the step/budget caps close it.
|
|
613
|
+
*
|
|
614
|
+
* A MutationObserver watches for the walk's whole reason to exist: the
|
|
615
|
+
* page REACTING to scroll by adding or removing elements. Most of the web
|
|
616
|
+
* never does (a long static article only costs latency here), so after
|
|
617
|
+
* {@link SCROLL_INERT_STEPS} careful steps with no reaction the walk
|
|
618
|
+
* probes the rest with one jump to the bottom. Any reaction after that
|
|
619
|
+
* jump — an uncloak, a re-cloak, a load-more append — means the region
|
|
620
|
+
* jumped over may hold content that never rendered, so the walk rewinds
|
|
621
|
+
* to where it jumped from and finishes carefully. Correctness never
|
|
622
|
+
* rests on the warm-up guess; only the cost of a scroll-inert page does.
|
|
623
|
+
* 4. MERGE into a structural clone (`document.cloneNode(true)`: no
|
|
624
|
+
* serialize/re-parse round trip — a runtime-rewritten data script with a
|
|
625
|
+
* literal `<\/script>` in its text cannot corrupt it — and, having no
|
|
626
|
+
* browsing context, it runs no scripts, upgrades no custom elements and
|
|
627
|
+
* loads no frames):
|
|
628
|
+
* - SPLICE (container walks): stashed keys from inside the scroll
|
|
629
|
+
* container that the clone no longer holds were REMOVED by the engine
|
|
630
|
+
* (the Feishu shape — its placeholders are anonymous) and go back next
|
|
631
|
+
* to their recorded sibling, resolved iteratively so a removed run
|
|
632
|
+
* chains off whichever end survived; a key that rode in inside its
|
|
633
|
+
* spliced parent is already present and skipped. Placement is
|
|
634
|
+
* structural, so it is right whatever order the walk SAW rows in (an
|
|
635
|
+
* anchored open starts mid-document).
|
|
636
|
+
* - SWAP: every keyed element emptier than its snapshot (fewer child
|
|
637
|
+
* elements, or meaningfully less text — the Discourse placeholder) is
|
|
638
|
+
* replaced by that snapshot, except containers (ancestors of another
|
|
639
|
+
* element being swapped), which are assembled from those descendants
|
|
640
|
+
* instead: a container's own snapshot is one window's partial view.
|
|
641
|
+
* The clone is then stripped of scripting elements and serialized; the
|
|
642
|
+
* provider reads that string instead of the live page. The live document
|
|
643
|
+
* is never rewritten, so no late scroll/rAF/timer reaction can touch the
|
|
644
|
+
* capture either.
|
|
645
|
+
*
|
|
646
|
+
* Failure is always silent degradation: a page without `evaluate`, a script
|
|
647
|
+
* error, or an engine that rejects the walk just keeps the legacy
|
|
648
|
+
* single-instant capture — a walk can only add content, never fail a fetch.
|
|
649
|
+
* A walk that restored nothing returns no HTML, and the provider reads the
|
|
650
|
+
* live page exactly as before.
|
|
651
|
+
*
|
|
652
|
+
* The walk runs in BOTH denoise modes (the raw-HTML path serializes the same
|
|
653
|
+
* cloaked placeholders) but never for non-HTML documents. It is bounded by
|
|
654
|
+
* the `scrollBudgetMs` setting (`0` restores the pre-walk legacy behavior)
|
|
655
|
+
* and always by the remaining fetch deadline minus a decode reserve — the
|
|
656
|
+
* provider's abort closes the tab, which rejects the pending evaluation.
|
|
657
|
+
*
|
|
658
|
+
* @module dsh-web-fetch-playwright/autoscroll
|
|
659
|
+
*/
|
|
660
|
+
/** How far each step scrolls, as a share of the viewport height. */
|
|
661
|
+
const SCROLL_STEP_RATIO = .7;
|
|
662
|
+
/**
|
|
663
|
+
* How long at the bottom, with no height growth, before the walk concludes
|
|
664
|
+
* the page is complete. Real feed engines fetch their next chunk with
|
|
665
|
+
* network latency (measured on linux.do: a pending chunk landed well after
|
|
666
|
+
* two 450ms checks), so this is sized over a slow round trip.
|
|
667
|
+
*/
|
|
668
|
+
const SCROLL_BOTTOM_PATIENCE_MS = 2500;
|
|
669
|
+
/**
|
|
670
|
+
* Cap on stashed keys — pathological id counts cannot pin memory. Sized for
|
|
671
|
+
* long topics: a Discourse post carries several ids of its own (post,
|
|
672
|
+
* heading, avatar, …), so a few hundred posts already need a thousand-plus.
|
|
673
|
+
* {@link SCROLL_STASH_MAX_CHARS} is the real memory bound.
|
|
674
|
+
*/
|
|
675
|
+
const SCROLL_STASH_MAX_ENTRIES = 2e3;
|
|
676
|
+
/** Cap on stashed HTML characters (aligned with the pipeline input cap). */
|
|
677
|
+
const SCROLL_STASH_MAX_CHARS = 2e6;
|
|
678
|
+
/**
|
|
679
|
+
* Cap on ONE stashed element's HTML. A virtualized row is a few KB; an
|
|
680
|
+
* element tens of KB big is a CONTAINER (measured on linux.do: six nested
|
|
681
|
+
* wrappers each embed the whole post stream — main 164KB ⊃
|
|
682
|
+
* main-outlet-wrapper 156KB ⊃ … — plus data-preloaded 300KB and
|
|
683
|
+
* svg-sprites 277KB) or a data island — never a row the merge could
|
|
684
|
+
* rehydrate. Such keys are remembered and never serialized again.
|
|
685
|
+
*/
|
|
686
|
+
const SCROLL_STASH_MAX_ENTRY_CHARS = 64e3;
|
|
687
|
+
/**
|
|
688
|
+
* Tail of the fetch budget the walk never spends, keeping room for the
|
|
689
|
+
* content read and the synchronous denoise decode.
|
|
690
|
+
*/
|
|
691
|
+
const SCROLL_FINISH_RESERVE_MS = 4e3;
|
|
692
|
+
/** The viewport share an inner container's scrollable distance must reach. */
|
|
693
|
+
const SCROLL_CONTAINER_MIN_RANGE_RATIO = .5;
|
|
694
|
+
/**
|
|
695
|
+
* The in-page walk. Evaluated BY SOURCE (`fn.toString()` composed into an
|
|
696
|
+
* expression by {@link deepCaptureScript}), so it must stay self-contained:
|
|
697
|
+
* every helper is nested, every knob arrives in `p`, and the one shared
|
|
698
|
+
* routine — the scripting strip — arrives as an argument. Typed here so the
|
|
699
|
+
* compiler checks what used to be an unchecked string.
|
|
700
|
+
*
|
|
701
|
+
* @param p - the walk's knobs.
|
|
702
|
+
* @param strip - {@link stripScriptingElementsIn}, applied to the merged clone.
|
|
703
|
+
* @returns the walk's stats, plus the merged HTML when anything was restored.
|
|
704
|
+
*/
|
|
705
|
+
async function deepCaptureInPage(p, strip) {
|
|
706
|
+
const started = Date.now();
|
|
707
|
+
const root = document.documentElement;
|
|
708
|
+
const body = document.body;
|
|
709
|
+
if (root === null || body === null) return {
|
|
710
|
+
steps: 0,
|
|
711
|
+
stashed: 0,
|
|
712
|
+
rehydrated: 0,
|
|
713
|
+
skipped: "not-scrollable"
|
|
714
|
+
};
|
|
715
|
+
if (p.budgetMs <= 0) return {
|
|
716
|
+
steps: 0,
|
|
717
|
+
stashed: 0,
|
|
718
|
+
rehydrated: 0,
|
|
719
|
+
skipped: "no-budget"
|
|
720
|
+
};
|
|
721
|
+
const KEYED = "[id], [data-block-id]";
|
|
722
|
+
const SVG_NS = "http://www.w3.org/2000/svg";
|
|
723
|
+
const sleep = (ms) => new Promise((resolve) => {
|
|
724
|
+
setTimeout(resolve, ms);
|
|
725
|
+
});
|
|
726
|
+
function findScrollContainer(host) {
|
|
727
|
+
const minRange = Math.max(p.containerMinRangePx, window.innerHeight * p.containerMinRatio);
|
|
728
|
+
let best = null;
|
|
729
|
+
let bestRange = 0;
|
|
730
|
+
for (const el of Array.from(host.querySelectorAll("*"))) {
|
|
731
|
+
const range = el.scrollHeight - el.clientHeight;
|
|
732
|
+
if (range < minRange || range <= bestRange) continue;
|
|
733
|
+
const overflowY = getComputedStyle(el).overflowY;
|
|
734
|
+
if (overflowY === "auto" || overflowY === "scroll") {
|
|
735
|
+
best = el;
|
|
736
|
+
bestRange = range;
|
|
737
|
+
}
|
|
738
|
+
}
|
|
739
|
+
return best;
|
|
740
|
+
}
|
|
741
|
+
const documentScrolls = root.scrollHeight > window.innerHeight + 4;
|
|
742
|
+
const scroller = documentScrolls ? null : findScrollContainer(body);
|
|
743
|
+
if (!documentScrolls && scroller === null) return {
|
|
744
|
+
steps: 0,
|
|
745
|
+
stashed: 0,
|
|
746
|
+
rehydrated: 0,
|
|
747
|
+
skipped: "not-scrollable"
|
|
748
|
+
};
|
|
749
|
+
const viewportH = () => scroller === null ? window.innerHeight : scroller.clientHeight;
|
|
750
|
+
const scrollH = () => scroller === null ? root.scrollHeight : scroller.scrollHeight;
|
|
751
|
+
const curY = () => scroller === null ? window.scrollY : scroller.scrollTop;
|
|
752
|
+
const atBottom = () => curY() + viewportH() >= scrollH() - 2;
|
|
753
|
+
function scrollToY(y) {
|
|
754
|
+
if (scroller === null) window.scrollTo({
|
|
755
|
+
top: y,
|
|
756
|
+
behavior: "instant"
|
|
757
|
+
});
|
|
758
|
+
else scroller.scrollTop = y;
|
|
759
|
+
}
|
|
760
|
+
function smoothScrollBy(dy) {
|
|
761
|
+
return new Promise((resolve) => {
|
|
762
|
+
const fromY = curY();
|
|
763
|
+
const t0 = Date.now();
|
|
764
|
+
const tick = () => {
|
|
765
|
+
const progress = (Date.now() - t0) / p.stepMotionMs;
|
|
766
|
+
if (progress >= 1 || curY() !== fromY && atBottom()) {
|
|
767
|
+
scrollToY(fromY + dy);
|
|
768
|
+
resolve();
|
|
769
|
+
return;
|
|
770
|
+
}
|
|
771
|
+
scrollToY(fromY + Math.round(dy * progress));
|
|
772
|
+
setTimeout(tick, 16);
|
|
773
|
+
};
|
|
774
|
+
tick();
|
|
775
|
+
});
|
|
776
|
+
}
|
|
777
|
+
function keyOf(el) {
|
|
778
|
+
if (el.id !== "") return el.id;
|
|
779
|
+
const block = el.getAttribute("data-block-id");
|
|
780
|
+
return block !== null && block !== "" ? `data-block-id:${block}` : null;
|
|
781
|
+
}
|
|
782
|
+
function siblingKey(el, dir) {
|
|
783
|
+
for (let sib = el[dir]; sib !== null; sib = sib[dir]) {
|
|
784
|
+
const key = keyOf(sib);
|
|
785
|
+
if (key !== null) return key;
|
|
786
|
+
}
|
|
787
|
+
return null;
|
|
788
|
+
}
|
|
789
|
+
const INERT_TAGS = /* @__PURE__ */ new Set([
|
|
790
|
+
"SCRIPT",
|
|
791
|
+
"STYLE",
|
|
792
|
+
"LINK",
|
|
793
|
+
"META",
|
|
794
|
+
"NOSCRIPT",
|
|
795
|
+
"IFRAME",
|
|
796
|
+
"TEMPLATE"
|
|
797
|
+
]);
|
|
798
|
+
let reactions = 0;
|
|
799
|
+
const counts = (nodes) => {
|
|
800
|
+
for (const node of Array.from(nodes)) if (node.nodeType === 1 && !INERT_TAGS.has(node.nodeName)) return true;
|
|
801
|
+
return false;
|
|
802
|
+
};
|
|
803
|
+
const tally = (records) => {
|
|
804
|
+
for (const record of records) if (counts(record.addedNodes) || counts(record.removedNodes)) reactions++;
|
|
805
|
+
};
|
|
806
|
+
const observer = new MutationObserver(tally);
|
|
807
|
+
observer.observe(body, {
|
|
808
|
+
childList: true,
|
|
809
|
+
subtree: true
|
|
810
|
+
});
|
|
811
|
+
const reacted = () => {
|
|
812
|
+
tally(observer.takeRecords());
|
|
813
|
+
return reactions > 0;
|
|
814
|
+
};
|
|
815
|
+
const stashDoc = document.implementation.createHTMLDocument("");
|
|
816
|
+
const stash = /* @__PURE__ */ new Map();
|
|
817
|
+
const oversized = /* @__PURE__ */ new Set();
|
|
818
|
+
let stashChars = 0;
|
|
819
|
+
const host = body;
|
|
820
|
+
function collect() {
|
|
821
|
+
for (const el of Array.from(host.querySelectorAll(KEYED))) {
|
|
822
|
+
const key = keyOf(el);
|
|
823
|
+
if (key === null || oversized.has(key) || el.namespaceURI === SVG_NS) continue;
|
|
824
|
+
const els = el.childElementCount;
|
|
825
|
+
const text = (el.textContent ?? "").length;
|
|
826
|
+
if (els === 0 && text === 0) continue;
|
|
827
|
+
const prev = stash.get(key);
|
|
828
|
+
if (prev !== void 0) {
|
|
829
|
+
prev.prev = siblingKey(el, "previousElementSibling") ?? prev.prev;
|
|
830
|
+
prev.next = siblingKey(el, "nextElementSibling") ?? prev.next;
|
|
831
|
+
if (els <= prev.els && text <= prev.text) continue;
|
|
832
|
+
} else if (stash.size >= p.stashMaxEntries) continue;
|
|
833
|
+
if (text > p.stashMaxEntryChars) {
|
|
834
|
+
oversized.add(key);
|
|
835
|
+
continue;
|
|
836
|
+
}
|
|
837
|
+
const len = el.outerHTML.length;
|
|
838
|
+
if (len > p.stashMaxEntryChars) {
|
|
839
|
+
oversized.add(key);
|
|
840
|
+
continue;
|
|
841
|
+
}
|
|
842
|
+
const freed = prev === void 0 ? 0 : prev.len;
|
|
843
|
+
if (stashChars - freed + len > p.stashMaxChars) continue;
|
|
844
|
+
stashChars += len - freed;
|
|
845
|
+
stash.set(key, {
|
|
846
|
+
node: stashDoc.importNode(el, true),
|
|
847
|
+
els,
|
|
848
|
+
text,
|
|
849
|
+
len,
|
|
850
|
+
prev: prev?.prev ?? siblingKey(el, "previousElementSibling"),
|
|
851
|
+
next: prev?.next ?? siblingKey(el, "nextElementSibling"),
|
|
852
|
+
inScroller: scroller !== null && scroller.contains(el)
|
|
853
|
+
});
|
|
854
|
+
}
|
|
855
|
+
}
|
|
856
|
+
collect();
|
|
857
|
+
if (curY() > 0) {
|
|
858
|
+
scrollToY(0);
|
|
859
|
+
await sleep(p.stepSettleMs);
|
|
860
|
+
collect();
|
|
861
|
+
}
|
|
862
|
+
tally(observer.takeRecords());
|
|
863
|
+
reactions = 0;
|
|
864
|
+
let steps = 0;
|
|
865
|
+
let lastHeight = scrollH();
|
|
866
|
+
let lastChangeAt = Date.now();
|
|
867
|
+
let careful = false;
|
|
868
|
+
let jumpedFrom = null;
|
|
869
|
+
const noteHeight = () => {
|
|
870
|
+
const height = scrollH();
|
|
871
|
+
if (height !== lastHeight) {
|
|
872
|
+
lastHeight = height;
|
|
873
|
+
lastChangeAt = Date.now();
|
|
874
|
+
}
|
|
875
|
+
};
|
|
876
|
+
try {
|
|
877
|
+
while (steps < p.maxSteps && Date.now() - started < p.budgetMs) {
|
|
878
|
+
if (jumpedFrom !== null && reacted()) {
|
|
879
|
+
scrollToY(jumpedFrom);
|
|
880
|
+
jumpedFrom = null;
|
|
881
|
+
careful = true;
|
|
882
|
+
await sleep(p.stepSettleMs);
|
|
883
|
+
collect();
|
|
884
|
+
noteHeight();
|
|
885
|
+
continue;
|
|
886
|
+
}
|
|
887
|
+
if (atBottom()) {
|
|
888
|
+
await sleep(p.bottomPollMs);
|
|
889
|
+
collect();
|
|
890
|
+
const before = lastHeight;
|
|
891
|
+
noteHeight();
|
|
892
|
+
if (lastHeight === before && Date.now() - lastChangeAt >= p.bottomPatienceMs) break;
|
|
893
|
+
continue;
|
|
894
|
+
}
|
|
895
|
+
if (!careful && jumpedFrom === null) {
|
|
896
|
+
if (reacted()) careful = true;
|
|
897
|
+
else if (steps >= p.inertSteps) {
|
|
898
|
+
jumpedFrom = curY();
|
|
899
|
+
steps++;
|
|
900
|
+
scrollToY(scrollH());
|
|
901
|
+
lastChangeAt = Date.now();
|
|
902
|
+
await sleep(p.stepSettleMs);
|
|
903
|
+
collect();
|
|
904
|
+
noteHeight();
|
|
905
|
+
continue;
|
|
906
|
+
}
|
|
907
|
+
}
|
|
908
|
+
steps++;
|
|
909
|
+
await smoothScrollBy(Math.max(1, Math.round(viewportH() * p.stepRatio)));
|
|
910
|
+
collect();
|
|
911
|
+
await sleep(p.stepSettleMs);
|
|
912
|
+
collect();
|
|
913
|
+
noteHeight();
|
|
914
|
+
}
|
|
915
|
+
} finally {
|
|
916
|
+
observer.disconnect();
|
|
917
|
+
}
|
|
918
|
+
const reactive = reacted();
|
|
919
|
+
const finalHeight = scrollH();
|
|
920
|
+
const fresh = document.cloneNode(true);
|
|
921
|
+
const present = /* @__PURE__ */ new Map();
|
|
922
|
+
const index = (node) => {
|
|
923
|
+
const key = keyOf(node);
|
|
924
|
+
if (key !== null && !present.has(key)) present.set(key, node);
|
|
925
|
+
for (const el of Array.from(node.querySelectorAll(KEYED))) {
|
|
926
|
+
const inner = keyOf(el);
|
|
927
|
+
if (inner !== null && !present.has(inner)) present.set(inner, el);
|
|
928
|
+
}
|
|
929
|
+
};
|
|
930
|
+
if (fresh.documentElement !== null) index(fresh.documentElement);
|
|
931
|
+
let spliced = 0;
|
|
932
|
+
if (scroller !== null) {
|
|
933
|
+
let pending = Array.from(stash.keys()).filter((key) => stash.get(key).inScroller && !present.has(key));
|
|
934
|
+
let progressed = true;
|
|
935
|
+
while (pending.length > 0 && progressed) {
|
|
936
|
+
progressed = false;
|
|
937
|
+
const unresolved = [];
|
|
938
|
+
for (const key of pending) {
|
|
939
|
+
if (present.has(key)) {
|
|
940
|
+
progressed = true;
|
|
941
|
+
continue;
|
|
942
|
+
}
|
|
943
|
+
const snap = stash.get(key);
|
|
944
|
+
const after = snap.prev === null ? void 0 : present.get(snap.prev);
|
|
945
|
+
const before = snap.next === null ? void 0 : present.get(snap.next);
|
|
946
|
+
if (after === void 0 && before === void 0) {
|
|
947
|
+
unresolved.push(key);
|
|
948
|
+
continue;
|
|
949
|
+
}
|
|
950
|
+
const node = fresh.importNode(snap.node, true);
|
|
951
|
+
if (after !== void 0) after.after(node);
|
|
952
|
+
else before.before(node);
|
|
953
|
+
index(node);
|
|
954
|
+
spliced++;
|
|
955
|
+
progressed = true;
|
|
956
|
+
}
|
|
957
|
+
pending = unresolved;
|
|
958
|
+
}
|
|
959
|
+
}
|
|
960
|
+
const degraded = [];
|
|
961
|
+
for (const el of Array.from(fresh.querySelectorAll(KEYED))) {
|
|
962
|
+
const key = keyOf(el);
|
|
963
|
+
const snap = key === null ? void 0 : stash.get(key);
|
|
964
|
+
if (snap === void 0 || el.namespaceURI === SVG_NS) continue;
|
|
965
|
+
if (el.childElementCount < snap.els || snap.text - (el.textContent ?? "").length > p.textDegradeTolerance) degraded.push({
|
|
966
|
+
el,
|
|
967
|
+
snap
|
|
968
|
+
});
|
|
969
|
+
}
|
|
970
|
+
const containers = /* @__PURE__ */ new Set();
|
|
971
|
+
for (const { el } of degraded) for (let up = el.parentElement; up !== null && !containers.has(up); up = up.parentElement) containers.add(up);
|
|
972
|
+
let rehydrated = 0;
|
|
973
|
+
for (const { el, snap } of degraded) {
|
|
974
|
+
if (containers.has(el)) continue;
|
|
975
|
+
el.replaceWith(fresh.importNode(snap.node, true));
|
|
976
|
+
rehydrated++;
|
|
977
|
+
}
|
|
978
|
+
const stats = {
|
|
979
|
+
steps,
|
|
980
|
+
stashed: stash.size,
|
|
981
|
+
rehydrated,
|
|
982
|
+
spliced,
|
|
983
|
+
reactive,
|
|
984
|
+
elapsedMs: Date.now() - started,
|
|
985
|
+
height: finalHeight
|
|
986
|
+
};
|
|
987
|
+
if ((rehydrated > 0 || spliced > 0) && fresh.documentElement !== null) {
|
|
988
|
+
strip(fresh);
|
|
989
|
+
stats.html = (fresh.doctype === null ? "" : new XMLSerializer().serializeToString(fresh.doctype)) + fresh.documentElement.outerHTML;
|
|
990
|
+
}
|
|
991
|
+
return stats;
|
|
992
|
+
}
|
|
993
|
+
/**
|
|
994
|
+
* The walk's knobs for one fetch: the module constants plus its budget.
|
|
995
|
+
*
|
|
996
|
+
* @param budgetMs - wall-clock budget the walk may spend inside the page.
|
|
997
|
+
*/
|
|
998
|
+
function deepCaptureParams(budgetMs) {
|
|
999
|
+
return {
|
|
1000
|
+
budgetMs: Math.max(0, Math.round(budgetMs)),
|
|
1001
|
+
stepRatio: SCROLL_STEP_RATIO,
|
|
1002
|
+
stepMotionMs: 300,
|
|
1003
|
+
stepSettleMs: 300,
|
|
1004
|
+
bottomPollMs: 450,
|
|
1005
|
+
bottomPatienceMs: SCROLL_BOTTOM_PATIENCE_MS,
|
|
1006
|
+
inertSteps: 2,
|
|
1007
|
+
maxSteps: 50,
|
|
1008
|
+
stashMaxEntries: SCROLL_STASH_MAX_ENTRIES,
|
|
1009
|
+
stashMaxChars: SCROLL_STASH_MAX_CHARS,
|
|
1010
|
+
stashMaxEntryChars: SCROLL_STASH_MAX_ENTRY_CHARS,
|
|
1011
|
+
containerMinRangePx: 300,
|
|
1012
|
+
containerMinRatio: SCROLL_CONTAINER_MIN_RANGE_RATIO,
|
|
1013
|
+
textDegradeTolerance: 16
|
|
1014
|
+
};
|
|
1015
|
+
}
|
|
1016
|
+
/**
|
|
1017
|
+
* Compose the evaluate expression: the in-page walk applied to `params` and
|
|
1018
|
+
* the shared scripting strip, both by source.
|
|
1019
|
+
*
|
|
1020
|
+
* @param params - the walk's knobs (see {@link deepCaptureParams}).
|
|
1021
|
+
* @returns the expression string to pass to `page.evaluate`.
|
|
1022
|
+
*/
|
|
1023
|
+
function deepCaptureExpression(params) {
|
|
1024
|
+
return `/* dsh-deep-capture */ (${deepCaptureInPage.toString()})(${JSON.stringify(params)}, ${stripScriptingElementsIn.toString()})`;
|
|
1025
|
+
}
|
|
1026
|
+
/** {@link deepCaptureExpression} over {@link deepCaptureParams} — one fetch's walk. */
|
|
1027
|
+
function deepCaptureScript(budgetMs) {
|
|
1028
|
+
return deepCaptureExpression(deepCaptureParams(budgetMs));
|
|
1029
|
+
}
|
|
1030
|
+
/**
|
|
1031
|
+
* Run the deep-capture walk in the page, degrading silently to the legacy
|
|
1032
|
+
* single-instant capture on any failure: a page without `evaluate`, a budget
|
|
1033
|
+
* too small to act, a rejected evaluation (the fetch deadline closes the tab
|
|
1034
|
+
* mid-walk), or a result that is not the stats object all just skip it. A
|
|
1035
|
+
* walk can only add content, never fail a fetch.
|
|
1036
|
+
*
|
|
1037
|
+
* @param page - the fetch's tab.
|
|
1038
|
+
* @param budgetMs - wall-clock budget for the in-page walk; budgets under
|
|
1039
|
+
* {@link SCROLL_MIN_BUDGET_MS} skip it.
|
|
1040
|
+
* @returns the walk's stats (with the merged `html` when it restored
|
|
1041
|
+
* anything), or undefined when the walk did not run.
|
|
1042
|
+
*/
|
|
1043
|
+
async function deepCaptureWalk(page, budgetMs) {
|
|
1044
|
+
if (typeof page.evaluate !== "function") return void 0;
|
|
1045
|
+
if (budgetMs < 500) return void 0;
|
|
1046
|
+
try {
|
|
1047
|
+
const result = await page.evaluate(deepCaptureScript(budgetMs));
|
|
1048
|
+
if (result === null || typeof result !== "object") return void 0;
|
|
1049
|
+
const raw = result;
|
|
1050
|
+
if (typeof raw.steps !== "number" || typeof raw.stashed !== "number" || typeof raw.rehydrated !== "number") return;
|
|
1051
|
+
return {
|
|
1052
|
+
steps: raw.steps,
|
|
1053
|
+
stashed: raw.stashed,
|
|
1054
|
+
rehydrated: raw.rehydrated,
|
|
1055
|
+
...typeof raw.spliced === "number" ? { spliced: raw.spliced } : {},
|
|
1056
|
+
...typeof raw.reactive === "boolean" ? { reactive: raw.reactive } : {},
|
|
1057
|
+
...typeof raw.elapsedMs === "number" ? { elapsedMs: raw.elapsedMs } : {},
|
|
1058
|
+
...typeof raw.height === "number" ? { height: raw.height } : {},
|
|
1059
|
+
...raw.skipped === "not-scrollable" || raw.skipped === "no-budget" ? { skipped: raw.skipped } : {},
|
|
1060
|
+
...typeof raw.html === "string" ? { html: raw.html } : {}
|
|
1061
|
+
};
|
|
1062
|
+
} catch {
|
|
1063
|
+
return;
|
|
1064
|
+
}
|
|
1065
|
+
}
|
|
1066
|
+
//#endregion
|
|
455
1067
|
//#region src/session-window.ts
|
|
456
1068
|
const ownership = /* @__PURE__ */ new WeakMap();
|
|
457
1069
|
const unavailable = /* @__PURE__ */ new WeakSet();
|
|
@@ -1092,11 +1704,14 @@ function installBuiltinSlashResolveCompat() {
|
|
|
1092
1704
|
* footers, and ad chrome by scoring link density and text mass) — its result
|
|
1093
1705
|
* must then pass a coverage gate that distrusts tiny or sliver-thin
|
|
1094
1706
|
* extractions, which on list-shaped pages (portals, catalogs) is exactly
|
|
1095
|
-
* what the link-density penalty leaves behind;
|
|
1096
|
-
*
|
|
1097
|
-
*
|
|
1098
|
-
*
|
|
1099
|
-
*
|
|
1707
|
+
* what the link-density penalty leaves behind; an accepted extraction is
|
|
1708
|
+
* then pruned of trailing link-dense sections — the post-article
|
|
1709
|
+
* recommendation rails and share bars the scorer's vocabularies cannot name
|
|
1710
|
+
* (they arrive as bare divs with no class to match); DOMPurify sanitizes
|
|
1711
|
+
* whatever HTML remains and forbids the layout tags noise lives in;
|
|
1712
|
+
* Turndown with the GFM plugin converts to markdown using the same style
|
|
1713
|
+
* options and span-safe table rules as the shipped `dsh-tool-web` renderer,
|
|
1714
|
+
* so output is consistent with what `web_fetch` produces elsewhere.
|
|
1100
1715
|
*
|
|
1101
1716
|
* Synchronous for its callers and unit-tested against fixture pages: jsdom
|
|
1102
1717
|
* itself is loaded lazily on the first call — and, inside a dsh profile, only
|
|
@@ -1336,6 +1951,209 @@ turndown.addRule("tableRowWithoutSpanExpansion", {
|
|
|
1336
1951
|
}
|
|
1337
1952
|
});
|
|
1338
1953
|
/**
|
|
1954
|
+
* Count non-whitespace characters under `root` that sit inside an anchor,
|
|
1955
|
+
* skipping the same noise subtrees {@link visibleTextLength} skips so the
|
|
1956
|
+
* numerator and denominator of a link-density ratio stay comparable.
|
|
1957
|
+
*
|
|
1958
|
+
* @param root - the subtree to measure; `null` counts as nothing.
|
|
1959
|
+
* @returns the number of non-whitespace characters inside `<a>` descendants.
|
|
1960
|
+
*/
|
|
1961
|
+
function linkTextLength(root) {
|
|
1962
|
+
if (root === null) return 0;
|
|
1963
|
+
let length = 0;
|
|
1964
|
+
const visit = (node, insideAnchor) => {
|
|
1965
|
+
if (node.nodeType === 3) {
|
|
1966
|
+
if (insideAnchor) length += (node.nodeValue ?? "").replace(/\s+/g, "").length;
|
|
1967
|
+
return;
|
|
1968
|
+
}
|
|
1969
|
+
if (node.nodeType !== 1 || NOISE_SUBTREE_TAGS.has(node.nodeName.toLowerCase())) return;
|
|
1970
|
+
const anchor = insideAnchor || node.nodeName.toLowerCase() === "a";
|
|
1971
|
+
for (let child = node.firstChild; child !== null; child = child.nextSibling) visit(child, anchor);
|
|
1972
|
+
};
|
|
1973
|
+
visit(root, false);
|
|
1974
|
+
return length;
|
|
1975
|
+
}
|
|
1976
|
+
/**
|
|
1977
|
+
* Minimum non-whitespace characters a block must carry before its link
|
|
1978
|
+
* density is even evaluated. Blocks below this are spacers, empty wrappers,
|
|
1979
|
+
* or one-line sign-offs — too small to be worth a removal decision, so the
|
|
1980
|
+
* trailing walk skips past them instead of stopping at them.
|
|
1981
|
+
*/
|
|
1982
|
+
const TRAILING_MIN_TEXT_CHARS = 40;
|
|
1983
|
+
/**
|
|
1984
|
+
* Link-density share above which a trailing block reads as a link farm:
|
|
1985
|
+
* recommendation feeds, related-article rails, tag clouds, share bars.
|
|
1986
|
+
* Measured on the 163.com dy-article badcase the section's pick carried 84
|
|
1987
|
+
* linked characters inside 183 total (0.46) — recommendation cards dilute
|
|
1988
|
+
* their density with author/date lines, so a 0.5 bar would miss them by
|
|
1989
|
+
* construction; an honest closing paragraph with a citation or two measures
|
|
1990
|
+
* 0.1–0.3, leaving 0.4 in the empty band between.
|
|
1991
|
+
*/
|
|
1992
|
+
const TRAILING_LINK_DENSITY = .4;
|
|
1993
|
+
/**
|
|
1994
|
+
* A trailing block qualifies as a link farm only when it is link-heavy in
|
|
1995
|
+
* aggregate too: at least {@link TRAILING_LINK_COUNT} anchors or
|
|
1996
|
+
* {@link TRAILING_LINK_TEXT_CHARS} linked characters. Density alone would
|
|
1997
|
+
* let a single long linked sentence (an inline citation) drag a legitimate
|
|
1998
|
+
* paragraph under the knife.
|
|
1999
|
+
*/
|
|
2000
|
+
const TRAILING_LINK_COUNT = 3;
|
|
2001
|
+
const TRAILING_LINK_TEXT_CHARS = 60;
|
|
2002
|
+
/**
|
|
2003
|
+
* A trailing block may never hold more than this share of the whole
|
|
2004
|
+
* article's text. The prune targets *appended* widgets; a block that IS most
|
|
2005
|
+
* of the extraction says the scorer picked the widget, which is the coverage
|
|
2006
|
+
* gate's failure mode to handle, not this walk's.
|
|
2007
|
+
*/
|
|
2008
|
+
const TRAILING_MAX_ARTICLE_SHARE = .6;
|
|
2009
|
+
/**
|
|
2010
|
+
* A trailing run may never hold more than this share of its container's
|
|
2011
|
+
* text — the aggregate form of the guard above, and the one that keeps
|
|
2012
|
+
* list-shaped extractions whole. A recommendation rail is a *minority*
|
|
2013
|
+
* appended to real content (the 163 rail is 44% of its container); a
|
|
2014
|
+
* portal or feed whose extraction IS a list of link rows (measured on a
|
|
2015
|
+
* Hacker News homepage capture: ~91 trailing rows, every one individually
|
|
2016
|
+
* link-dense at 0.69–0.94) has a tail run holding ~97% of the container —
|
|
2017
|
+
* there the tail IS the content, the walk declines wholesale, and only a
|
|
2018
|
+
* run that is a minority of its container is removed.
|
|
2019
|
+
*/
|
|
2020
|
+
const TRAILING_MAX_RUN_SHARE = .6;
|
|
2021
|
+
/**
|
|
2022
|
+
* Link-density band within which a run and the block the walk stopped at
|
|
2023
|
+
* read as the same material. When the content immediately before the run
|
|
2024
|
+
* is itself as link-dense as the run — measured on a Hacker News homepage
|
|
2025
|
+
* capture, the rows after the run's cut measure 0.94 against the run's
|
|
2026
|
+
* 0.91 — the container is a *uniform list* (feed rows, portal headlines)
|
|
2027
|
+
* and the tail is that list's own content, not a widget appended after it;
|
|
2028
|
+
* a widget hangs after genuinely different material (the 163 rail's 0.44
|
|
2029
|
+
* follows body text at 0.00). Only a run denser than what precedes it is
|
|
2030
|
+
* removed.
|
|
2031
|
+
*/
|
|
2032
|
+
const TRAILING_UNIFORM_DENSITY_BAND = .25;
|
|
2033
|
+
/**
|
|
2034
|
+
* Prune trailing link-dense sections from an accepted Readability
|
|
2035
|
+
* extraction — the post-article widget class the scorer itself cannot be
|
|
2036
|
+
* taught to drop.
|
|
2037
|
+
*
|
|
2038
|
+
* Why this exists (badcase: `https://www.163.com/dy/article/…`): Chinese
|
|
2039
|
+
* portal article pages append a recommendation feed inside the very wrapper
|
|
2040
|
+
* Readability scores as the article (`#content` holds the body, the
|
|
2041
|
+
* platform disclaimer, then a rail of image/title/author/date cards). Every
|
|
2042
|
+
* existing noise layer misses it: Readability's `unlikelyCandidates` /
|
|
2043
|
+
* `negative` vocabularies know `related` but not `recommend` (nor the
|
|
2044
|
+
* pinyin families `tuijian`/`xiangguan`), and its own cleaning pass draws
|
|
2045
|
+
* the "mostly links" line at per-block density 0.5 — these cards measure
|
|
2046
|
+
* 0.42–0.48 (the title links over title+author+date text), so every card
|
|
2047
|
+
* clears it; in the extracted HTML the rail is a *bare* div — class and id
|
|
2048
|
+
* are gone — so no class-based rule can ever see it again; DOMPurify's
|
|
2049
|
+
* FORBID_TAGS works on tags, and the rail is plain div/p/h3/a; and the
|
|
2050
|
+
* coverage gate cannot help because the noise rides *inside* the
|
|
2051
|
+
* extraction it measures (the badcase passed at 0.103 — the rail's text
|
|
2052
|
+
* was in both numerator and denominator). Modern sites with CSS-module
|
|
2053
|
+
* class names (`css-1b3x7r`) present the same shape with no vocabulary to
|
|
2054
|
+
* match at all, so the only signal that survives all of this is
|
|
2055
|
+
* structural: a *trailing* run of blocks whose text is mostly links.
|
|
2056
|
+
*
|
|
2057
|
+
* For every container in the extraction the walk gathers the maximal
|
|
2058
|
+
* trailing run of qualifying children — link-farm blocks per the constants
|
|
2059
|
+
* above, plus sub-threshold spacers (empty wrapper divs) — and removes the
|
|
2060
|
+
* run only when BOTH structural guards say "appended widget, not content":
|
|
2061
|
+
* the run is a minority of the container's text
|
|
2062
|
+
* ({@link TRAILING_MAX_RUN_SHARE}), and the run is clearly denser than the
|
|
2063
|
+
* block the walk stopped at ({@link TRAILING_UNIFORM_DENSITY_BAND}). The
|
|
2064
|
+
* guards split the two worlds the per-block view cannot: a rail is
|
|
2065
|
+
* one-or-few appended blocks after genuinely different material (the 163
|
|
2066
|
+
* rail's 0.44 density follows body text at 0.00), while a link-list
|
|
2067
|
+
* extraction — feed rows, portal headlines — is *uniform*: its rows before
|
|
2068
|
+
* any cut measure the same density as the rows after it (0.91 vs 0.94 on
|
|
2069
|
+
* the Hacker News capture), so no tail of it can be mistaken for a widget.
|
|
2070
|
+
*
|
|
2071
|
+
* Deliberately NOT applied to the whole-document fallback: that path
|
|
2072
|
+
* exists for list-shaped pages where the links *are* the content (the
|
|
2073
|
+
* 163.com homepage fix), and pruning link-dense regions there would
|
|
2074
|
+
* re-introduce exactly the recall loss the coverage gate exists to
|
|
2075
|
+
* prevent.
|
|
2076
|
+
*
|
|
2077
|
+
* @param container - parsed `article.content`; mutated in place.
|
|
2078
|
+
*/
|
|
2079
|
+
function pruneTrailingLinkDenseSections(container) {
|
|
2080
|
+
const articleText = visibleTextLength(container);
|
|
2081
|
+
if (articleText === 0) return;
|
|
2082
|
+
for (const element of [container, ...Array.from(container.querySelectorAll("*"))]) {
|
|
2083
|
+
if (!container.contains(element) && element !== container) continue;
|
|
2084
|
+
removeTrailingLinkDenseRun(element, articleText);
|
|
2085
|
+
}
|
|
2086
|
+
}
|
|
2087
|
+
/**
|
|
2088
|
+
* Remove one container's maximal trailing run of link-dense children when
|
|
2089
|
+
* that run is a minority of the container's text AND clearly different
|
|
2090
|
+
* material from the content that precedes it.
|
|
2091
|
+
*
|
|
2092
|
+
* @param parent - the container whose element children are examined from
|
|
2093
|
+
* the end.
|
|
2094
|
+
* @param articleText - the whole article's text, for the per-block share
|
|
2095
|
+
* guard.
|
|
2096
|
+
*/
|
|
2097
|
+
function removeTrailingLinkDenseRun(parent, articleText) {
|
|
2098
|
+
const children = Array.from(parent.children);
|
|
2099
|
+
if (children.length === 0) return;
|
|
2100
|
+
const containerText = visibleTextLength(parent);
|
|
2101
|
+
if (containerText === 0) return;
|
|
2102
|
+
const run = [];
|
|
2103
|
+
let runText = 0;
|
|
2104
|
+
let runLinked = 0;
|
|
2105
|
+
let stopIndex = -1;
|
|
2106
|
+
for (let index = children.length - 1; index >= 0; index--) {
|
|
2107
|
+
const child = children[index];
|
|
2108
|
+
if (child === void 0) break;
|
|
2109
|
+
const text = visibleTextLength(child);
|
|
2110
|
+
if (text < TRAILING_MIN_TEXT_CHARS) {
|
|
2111
|
+
run.push(child);
|
|
2112
|
+
continue;
|
|
2113
|
+
}
|
|
2114
|
+
const linked = linkTextLength(child);
|
|
2115
|
+
const linkCount = child.querySelectorAll("a").length;
|
|
2116
|
+
if (linked / text >= TRAILING_LINK_DENSITY && (linkCount >= TRAILING_LINK_COUNT || linked >= TRAILING_LINK_TEXT_CHARS) && text <= TRAILING_MAX_ARTICLE_SHARE * articleText) {
|
|
2117
|
+
run.push(child);
|
|
2118
|
+
runText += text;
|
|
2119
|
+
runLinked += linked;
|
|
2120
|
+
continue;
|
|
2121
|
+
}
|
|
2122
|
+
stopIndex = index;
|
|
2123
|
+
break;
|
|
2124
|
+
}
|
|
2125
|
+
if (runText === 0 || stopIndex === -1) return;
|
|
2126
|
+
const stopChild = children[stopIndex];
|
|
2127
|
+
if (stopChild === void 0) return;
|
|
2128
|
+
const stopDensity = linkTextLength(stopChild) / Math.max(1, visibleTextLength(stopChild));
|
|
2129
|
+
if (Math.abs(runLinked / runText - stopDensity) <= TRAILING_UNIFORM_DENSITY_BAND) return;
|
|
2130
|
+
if (runText > TRAILING_MAX_RUN_SHARE * containerText) return;
|
|
2131
|
+
for (const child of run) child.remove();
|
|
2132
|
+
}
|
|
2133
|
+
/**
|
|
2134
|
+
* Run {@link pruneTrailingLinkDenseSections} over one extraction with a
|
|
2135
|
+
* misfire guard: a prune that takes a healthy article below the gate's
|
|
2136
|
+
* absolute floor ({@link GATE_MIN_ARTICLE_TEXT_CHARS}) or empties it
|
|
2137
|
+
* entirely says the walk misread the page, and Readability's original pick
|
|
2138
|
+
* is returned untouched. A *sparse* article (under the floor before the
|
|
2139
|
+
* prune, auto-trusted by the gate) keeps its pruned form — its trailing
|
|
2140
|
+
* links were noise all the same.
|
|
2141
|
+
*
|
|
2142
|
+
* @param articleHtml - the `article.content` HTML Readability produced.
|
|
2143
|
+
* @param document - the live document, for parsing the fragment.
|
|
2144
|
+
* @returns the pruned article HTML, or the original on a misfire.
|
|
2145
|
+
*/
|
|
2146
|
+
function prunedArticleContent(articleHtml, document) {
|
|
2147
|
+
const container = document.createElement("div");
|
|
2148
|
+
container.innerHTML = articleHtml;
|
|
2149
|
+
const before = visibleTextLength(container);
|
|
2150
|
+
if (before === 0) return articleHtml;
|
|
2151
|
+
pruneTrailingLinkDenseSections(container);
|
|
2152
|
+
const after = visibleTextLength(container);
|
|
2153
|
+
if (after === 0 || before >= GATE_MIN_ARTICLE_TEXT_CHARS && after < GATE_MIN_ARTICLE_TEXT_CHARS) return articleHtml;
|
|
2154
|
+
return container.innerHTML;
|
|
2155
|
+
}
|
|
2156
|
+
/**
|
|
1339
2157
|
* Convert one rendered HTML document to denoised markdown.
|
|
1340
2158
|
*
|
|
1341
2159
|
* Readability failure (non-article pages, or extractions the coverage gate
|
|
@@ -1363,7 +2181,7 @@ function htmlToMarkdown(html, url) {
|
|
|
1363
2181
|
const cloned = document.cloneNode(true);
|
|
1364
2182
|
const article = new Readability(cloned).parse();
|
|
1365
2183
|
if (article !== null && typeof article.content === "string" && article.content !== "" && articleCoversPage(article.content, document)) {
|
|
1366
|
-
source = article.content;
|
|
2184
|
+
source = prunedArticleContent(article.content, document);
|
|
1367
2185
|
title = article.title ?? void 0;
|
|
1368
2186
|
}
|
|
1369
2187
|
} catch {}
|
|
@@ -1621,6 +2439,31 @@ function existsAsFile(path) {
|
|
|
1621
2439
|
* fails with {@link WEB_FETCH_CHALLENGE_CODE} instead of returning the
|
|
1622
2440
|
* interstitial as content.
|
|
1623
2441
|
*
|
|
2442
|
+
* Deep capture (viewport-virtualized feeds and app-shell documents): feed-
|
|
2443
|
+
* shaped pages — Discourse topic streams, comment threads, timelines —
|
|
2444
|
+
* render only the items near the scroll position; everything else is an
|
|
2445
|
+
* empty, height-only placeholder that keeps the item's id but none of its
|
|
2446
|
+
* content, so a single-instant capture hands back the skeleton while the
|
|
2447
|
+
* bodies silently vanish (and the denoise pipeline's coverage gate cannot
|
|
2448
|
+
* notice: the page's own visible text is the same windowed fragment as the
|
|
2449
|
+
* extraction). App-shell document editors (Feishu docs) push the same
|
|
2450
|
+
* pathology one level down: the body never scrolls, blocks are keyed by
|
|
2451
|
+
* data-block-id with no id, and off-window blocks are REMOVED behind an
|
|
2452
|
+
* anonymous height spacer — the fetch would return just the statically
|
|
2453
|
+
* rendered first screen. Before serialization the fetch therefore runs one
|
|
2454
|
+
* bounded in-page scroll walk — over the window or, when the document does
|
|
2455
|
+
* not scroll, the page's dominant inner scroll container — stashing what
|
|
2456
|
+
* each rendered window shows, terminating only AT the bottom with the
|
|
2457
|
+
* height stable (that is what pages load-more sentinels; height stability
|
|
2458
|
+
* elsewhere proves nothing), then rehydrating degraded keyed elements from
|
|
2459
|
+
* their stashed snapshots and splicing removed keys back at their document
|
|
2460
|
+
* position so the final DOM holds the union of every window in document
|
|
2461
|
+
* order. The existing pipeline then runs unchanged. Bounded by
|
|
2462
|
+
* `scrollBudgetMs` (default 20s; 0 restores the legacy single-instant
|
|
2463
|
+
* behavior) and by the remaining fetch deadline; any walk failure silently
|
|
2464
|
+
* degrades to the legacy capture (see `src/autoscroll.ts` for the measured
|
|
2465
|
+
* linux.do and Feishu badcases).
|
|
2466
|
+
*
|
|
1624
2467
|
* @module dsh-web-fetch-playwright/provider
|
|
1625
2468
|
*/
|
|
1626
2469
|
/** Stable id this provider registers under (the bundle patch pins it). */
|
|
@@ -1978,8 +2821,14 @@ var PlaywrightFetchProvider = class {
|
|
|
1978
2821
|
content: finalResponse !== null ? await finalResponse.text() : await page.content()
|
|
1979
2822
|
});
|
|
1980
2823
|
await page.waitForLoadState("networkidle", { timeout: Math.min(SETTLE_MS, deadline.remainingMs()) }).catch(() => {});
|
|
1981
|
-
|
|
1982
|
-
const
|
|
2824
|
+
const scrollBudgetMs = Math.min(effectiveScrollBudgetMs(config), deadline.remainingMs() - SCROLL_FINISH_RESERVE_MS);
|
|
2825
|
+
const captured = scrollBudgetMs > 0 ? await deepCaptureWalk(page, scrollBudgetMs) : void 0;
|
|
2826
|
+
let html;
|
|
2827
|
+
if (captured?.html !== void 0) html = captured.html;
|
|
2828
|
+
else {
|
|
2829
|
+
await stripScriptingElements(page);
|
|
2830
|
+
html = await page.content();
|
|
2831
|
+
}
|
|
1983
2832
|
if (!config.denoise) return capResult(finalUrl, statusCode, {
|
|
1984
2833
|
kind: "html",
|
|
1985
2834
|
content: html
|
|
@@ -2292,66 +3141,12 @@ async function probeStillOnChallenge(page) {
|
|
|
2292
3141
|
}
|
|
2293
3142
|
}
|
|
2294
3143
|
/**
|
|
2295
|
-
* In-page script that removes non-content scripting subtrees from the live
|
|
2296
|
-
* document before `page.content()` reads it as an HTML string.
|
|
2297
|
-
*
|
|
2298
|
-
* WHAT goes, and why each removal is output-neutral versus the old pipeline:
|
|
2299
|
-
*
|
|
2300
|
-
* - `script` — three consumers ever saw script text before, and none loses
|
|
2301
|
-
* it here: Readability's `parse()` deletes script/noscript nodes BEFORE
|
|
2302
|
-
* extraction and scoring (so script text never reached the article or
|
|
2303
|
-
* influenced the scorer); DOMPurify's default allowlist drops script; the
|
|
2304
|
-
* tool layer's renderer removes SCRIPT wholesale on the raw path. What the
|
|
2305
|
-
* removal buys is the reason this exists: a page's runtime JS may rewrite
|
|
2306
|
-
* a data script's text with a literal `<\/script>` embedded — measured on
|
|
2307
|
-
* techcommunity.microsoft.com (Khoros/Aurora), whose client writes its
|
|
2308
|
-
* Apollo cache back into `script#__NEXT_DATA__` via `JSON.stringify`,
|
|
2309
|
-
* turning the wire's 712 `\u003c` escapes back into raw `<` — and
|
|
2310
|
-
* browsers serialize script text verbatim, so the flattened string ends
|
|
2311
|
-
* that script at the FIRST embedded end tag and re-parses the remaining
|
|
2312
|
-
* ~270KB of JSON as markup. The phantom elements that survive (unknown
|
|
2313
|
-
* tags like `<id>`, anchors full of JSON text) are exactly what
|
|
2314
|
-
* Readability's scorer reads as article mass, so the extracted "article"
|
|
2315
|
-
* is CSS and i18n strings while the real body — still present in the DOM —
|
|
2316
|
-
* loses. No string-level patch can undo this after serialization (the
|
|
2317
|
-
* string is genuinely ambiguous); only the browser's own parse holds the
|
|
2318
|
-
* document's true structure.
|
|
2319
|
-
* - `script[type="application/ld+json"]` — KEPT, but defused: Readability
|
|
2320
|
-
* extracts metadata (title/byline/excerpt — e.g. the site-suffix-free
|
|
2321
|
-
* "Remote Desktop Protocol" instead of "… - Wikipedia") from JSON-LD
|
|
2322
|
-
* scripts before removing them, so deleting them would regress the title
|
|
2323
|
-
* on every JSON-LD-carrying page. Rewriting `</` as `<\/` keeps the text
|
|
2324
|
-
* valid JSON (`\/` is a legal escape, so the re-parsed text parses to the
|
|
2325
|
-
* same object) while making the serialization harmless: the tokenizer
|
|
2326
|
-
* only terminates on a literal `<\/script`.
|
|
2327
|
-
* - `noscript` — KEPT: Readability's `_unwrapNoscriptImages` salvages the
|
|
2328
|
-
* real image out of lazy-load `<noscript><img …>` wrappers before
|
|
2329
|
-
* `_removeScripts` deletes the noscript; pre-stripping it would silently
|
|
2330
|
-
* drop those images from the markdown.
|
|
2331
|
-
* - `template` — removed: its children parse into the inert `.content`
|
|
2332
|
-
* fragment (invisible to Readability's traversal), DOMPurify's
|
|
2333
|
-
* FORBID_TAGS drops it, and the tool layer's renderer removes it too.
|
|
2334
|
-
* - `style` stays — its text cannot terminate a re-parse except at its own
|
|
2335
|
-
* end tag, and the pipeline already drops it.
|
|
2336
|
-
*/
|
|
2337
|
-
const STRIP_SCRIPTING_ELEMENTS = `(() => {
|
|
2338
|
-
for (const el of Array.from(document.querySelectorAll('script, template'))) {
|
|
2339
|
-
if (el.tagName === 'SCRIPT' && (el.getAttribute('type') || '').trim().toLowerCase() === 'application/ld+json') {
|
|
2340
|
-
// '<\\/' is still valid JSON, so the re-parsed textContent parses to
|
|
2341
|
-
// the same object — while the serialized form can no longer close the
|
|
2342
|
-
// script element early.
|
|
2343
|
-
el.textContent = el.textContent.replace(/<\\//g, '<\\\\/')
|
|
2344
|
-
continue
|
|
2345
|
-
}
|
|
2346
|
-
el.remove()
|
|
2347
|
-
}
|
|
2348
|
-
})()`;
|
|
2349
|
-
/**
|
|
2350
3144
|
* Remove non-content scripting subtrees from the live document before
|
|
2351
|
-
* serialization
|
|
2352
|
-
*
|
|
2353
|
-
*
|
|
2354
|
-
*
|
|
3145
|
+
* serialization (see `stripScriptingElementsIn` in `src/strip.ts` for exactly
|
|
3146
|
+
* what goes and why). Best-effort, mirroring {@link probeStillOnChallenge}: a backend
|
|
3147
|
+
* without `evaluate` (fakes, minimal engines) simply keeps the document
|
|
3148
|
+
* as-is — the fetch never fails here, and the pipeline degrades to whatever
|
|
3149
|
+
* the serialized string itself says.
|
|
2355
3150
|
*/
|
|
2356
3151
|
async function stripScriptingElements(page) {
|
|
2357
3152
|
if (typeof page.evaluate !== "function") return;
|
|
@@ -2488,4 +3283,4 @@ function apply(ctx, config) {
|
|
|
2488
3283
|
ctx.web.registerFetchProvider(provider);
|
|
2489
3284
|
}
|
|
2490
3285
|
//#endregion
|
|
2491
|
-
export { CHALLENGE_DOM_PROBE, CHALLENGE_FINISH_RESERVE_MS, CHALLENGE_POLL_INTERVAL_MS, CHALLENGE_TITLE_RE, CdpConnectionPool, Config, DEFAULT_CDP_ENDPOINT, DEFAULT_CHALLENGE_RETRIES, DEFAULT_CHALLENGE_WAIT_MS, PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider, WEB_FETCH_CHALLENGE_CODE, WEB_FETCH_PLAYWRIGHT_SETTINGS_NAMESPACE, apply, classifyChallengeHtml, classifyChallengeResponse, effectiveChallengeRetries, effectiveChallengeWaitMs, effectiveContextMode, htmlToMarkdown, inject, isChallengeCompatibleResponse, name, normalizeCdpEndpoint, resolveConfig, snapshotsOf };
|
|
3286
|
+
export { CHALLENGE_DOM_PROBE, CHALLENGE_FINISH_RESERVE_MS, CHALLENGE_POLL_INTERVAL_MS, CHALLENGE_TITLE_RE, CdpConnectionPool, Config, DEFAULT_CDP_ENDPOINT, DEFAULT_CHALLENGE_RETRIES, DEFAULT_CHALLENGE_WAIT_MS, DEFAULT_SCROLL_BUDGET_MS, MAX_SCROLL_BUDGET_MS, PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider, WEB_FETCH_CHALLENGE_CODE, WEB_FETCH_PLAYWRIGHT_SETTINGS_NAMESPACE, apply, classifyChallengeHtml, classifyChallengeResponse, deepCaptureWalk, effectiveChallengeRetries, effectiveChallengeWaitMs, effectiveContextMode, effectiveScrollBudgetMs, htmlToMarkdown, inject, isChallengeCompatibleResponse, name, normalizeCdpEndpoint, resolveConfig, snapshotsOf };
|