dsh-web-fetch-playwright 0.2.19 → 0.2.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -15
- package/README.zh-CN.md +15 -15
- package/cordis.patch.yml +0 -4
- package/lib/client.js +23 -73
- package/lib/index.js +58188 -448
- package/lib/types/autoscroll.d.ts +46 -10
- package/lib/types/block-model.d.ts +2 -0
- package/lib/types/blocks.d.ts +67 -0
- package/lib/types/candidates.d.ts +164 -0
- package/lib/types/client/controller.d.ts +0 -14
- package/lib/types/client/locales.d.ts +1 -1
- package/lib/types/config.d.ts +54 -30
- package/lib/types/index.d.ts +5 -3
- package/lib/types/markdown.d.ts +50 -24
- package/lib/types/provider.d.ts +46 -9
- package/lib/types/selector-model.d.ts +2 -0
- package/lib/types/selector.d.ts +58 -0
- package/lib/types/trees.d.ts +25 -0
- package/lib/types/types.d.ts +7 -6
- package/package.json +1 -1
- package/src/autoscroll.ts +459 -17
- package/src/block-model.ts +5 -0
- package/src/blocks.ts +223 -0
- package/src/candidates.ts +655 -0
- package/src/client/card.tsx +0 -26
- package/src/client/controller.ts +0 -18
- package/src/client/locales.ts +0 -14
- package/src/config.ts +67 -38
- package/src/index.ts +6 -3
- package/src/markdown.ts +136 -413
- package/src/provider.ts +198 -37
- package/src/selector-model.ts +6 -0
- package/src/selector.ts +288 -0
- package/src/trees.ts +43 -0
- package/src/types.ts +7 -7
|
@@ -60,7 +60,8 @@
|
|
|
60
60
|
* collecting after each, until the walk can hold AT THE BOTTOM through a
|
|
61
61
|
* {@link SCROLL_BOTTOM_PATIENCE_MS} window with the height unchanged
|
|
62
62
|
* (growth restarts the clock — that is what pages load-more sentinels),
|
|
63
|
-
* or the step/budget caps close it.
|
|
63
|
+
* or the step/budget caps close it. A page that never reacted and issued
|
|
64
|
+
* no fetch/XHR holds only {@link SCROLL_INERT_PATIENCE_MS}.
|
|
64
65
|
*
|
|
65
66
|
* A MutationObserver watches for the walk's whole reason to exist: the
|
|
66
67
|
* page REACTING to scroll by adding or removing elements. Most of the web
|
|
@@ -71,6 +72,14 @@
|
|
|
71
72
|
* jumped over may hold content that never rendered, so the walk rewinds
|
|
72
73
|
* to where it jumped from and finishes carefully. Correctness never
|
|
73
74
|
* rests on the warm-up guess; only the cost of a scroll-inert page does.
|
|
75
|
+
*
|
|
76
|
+
* The same observer HARVESTS UNKEYED ROWS — the third shape: react-window
|
|
77
|
+
* lists and X-style timelines give rows no DOM identity at all (the
|
|
78
|
+
* framework key never reaches the DOM), and framework-generated ids
|
|
79
|
+
* (`ember123`, React useId's `:r1:`) change per render so they are not
|
|
80
|
+
* keys either. When the engine removes such a row, the record still
|
|
81
|
+
* holds it whole plus the sibling nodes it sat between; that is the row's
|
|
82
|
+
* stash and its splice anchor, with no layout read at all.
|
|
74
83
|
* 4. MERGE into a structural clone (`document.cloneNode(true)`: no
|
|
75
84
|
* serialize/re-parse round trip — a runtime-rewritten data script with a
|
|
76
85
|
* literal `</script>` in its text cannot corrupt it — and, having no
|
|
@@ -84,11 +93,21 @@
|
|
|
84
93
|
* spliced parent is already present and skipped. Placement is
|
|
85
94
|
* structural, so it is right whatever order the walk SAW rows in (an
|
|
86
95
|
* anchored open starts mid-document).
|
|
96
|
+
* - RESTORE (unkeyed rows): in every list the engine both removed text-
|
|
97
|
+
* bearing rows from and added to during the walk, each removed row the
|
|
98
|
+
* list no longer holds goes back beside the sibling it was removed
|
|
99
|
+
* next to (chains through removed anchors; rows with no surviving
|
|
100
|
+
* anchor at all — a wholesale innerHTML re-render — go above the final
|
|
101
|
+
* window, which is the only place a top-down walk loses them). Rows
|
|
102
|
+
* still present are recognised by a per-list identity attribute
|
|
103
|
+
* (data-index, aria-rowindex, …) or by text; in-place stubs are
|
|
104
|
+
* absorbed.
|
|
87
105
|
* - SWAP: every keyed element emptier than its snapshot (fewer child
|
|
88
106
|
* elements, or meaningfully less text — the Discourse placeholder) is
|
|
89
107
|
* replaced by that snapshot, except containers (ancestors of another
|
|
90
108
|
* element being swapped), which are assembled from those descendants
|
|
91
|
-
* instead: a container's own snapshot is one window's partial view.
|
|
109
|
+
* instead: a container's own snapshot is one window's partial view. A
|
|
110
|
+
* restored list and its ancestors are containers too.
|
|
92
111
|
* The clone is then stripped of scripting elements and serialized; the
|
|
93
112
|
* provider reads that string instead of the live page. The live document
|
|
94
113
|
* is never rewritten, so no late scroll/rAF/timer reaction can touch the
|
|
@@ -98,13 +117,16 @@
|
|
|
98
117
|
* error, or an engine that rejects the walk just keeps the legacy
|
|
99
118
|
* single-instant capture — a walk can only add content, never fail a fetch.
|
|
100
119
|
* A walk that restored nothing returns no HTML, and the provider reads the
|
|
101
|
-
* live page exactly as before
|
|
120
|
+
* live page exactly as before — scrolled back to where it opened when the
|
|
121
|
+
* page reacted, so a virtualizer the merge could not rebuild hands over its
|
|
122
|
+
* opening window rather than the walk's last one.
|
|
102
123
|
*
|
|
103
124
|
* The walk runs in BOTH denoise modes (the raw-HTML path serializes the same
|
|
104
125
|
* cloaked placeholders) but never for non-HTML documents. It is bounded by
|
|
105
126
|
* the `scrollBudgetMs` setting (`0` restores the pre-walk legacy behavior)
|
|
106
|
-
* and always by the remaining
|
|
107
|
-
*
|
|
127
|
+
* and always by the fetch's remaining waiting budget: a walk cut short there
|
|
128
|
+
* still merges and returns what it stashed, and the content read plus decode
|
|
129
|
+
* run in the provider's extraction grace window after it.
|
|
108
130
|
*
|
|
109
131
|
* @module dsh-web-fetch-playwright/autoscroll
|
|
110
132
|
*/
|
|
@@ -135,6 +157,16 @@ export declare const SCROLL_BOTTOM_POLL_MS = 450;
|
|
|
135
157
|
* two 450ms checks), so this is sized over a slow round trip.
|
|
136
158
|
*/
|
|
137
159
|
export declare const SCROLL_BOTTOM_PATIENCE_MS = 2500;
|
|
160
|
+
/**
|
|
161
|
+
* Bottom hold for a page that has given no sign of loading more: it never
|
|
162
|
+
* added or removed an element in response to the walk's scrolling, and it
|
|
163
|
+
* issued no fetch/XHR since the walk began. The full
|
|
164
|
+
* {@link SCROLL_BOTTOM_PATIENCE_MS} exists for feeds whose next chunk is in
|
|
165
|
+
* flight; holding it on every static article made the walk cost ~4s on
|
|
166
|
+
* pages it could never help. Any reaction or data request (even one that
|
|
167
|
+
* lands during this shorter hold) restores the full patience.
|
|
168
|
+
*/
|
|
169
|
+
export declare const SCROLL_INERT_PATIENCE_MS = 900;
|
|
138
170
|
/**
|
|
139
171
|
* Careful steps without any DOM reaction before the walk treats the page as
|
|
140
172
|
* scroll-inert and probes the rest with a single jump (rewinding if the
|
|
@@ -162,11 +194,6 @@ export declare const SCROLL_STASH_MAX_CHARS = 2000000;
|
|
|
162
194
|
* rehydrate. Such keys are remembered and never serialized again.
|
|
163
195
|
*/
|
|
164
196
|
export declare const SCROLL_STASH_MAX_ENTRY_CHARS = 64000;
|
|
165
|
-
/**
|
|
166
|
-
* Tail of the fetch budget the walk never spends, keeping room for the
|
|
167
|
-
* content read and the synchronous denoise decode.
|
|
168
|
-
*/
|
|
169
|
-
export declare const SCROLL_FINISH_RESERVE_MS = 4000;
|
|
170
197
|
/**
|
|
171
198
|
* A walk budget below this cannot complete even the first window plus one
|
|
172
199
|
* stability check, so the walk is skipped outright.
|
|
@@ -195,6 +222,14 @@ export interface DeepCaptureStats {
|
|
|
195
222
|
rehydrated: number;
|
|
196
223
|
/** Removed keyed elements spliced back (container walks only). */
|
|
197
224
|
spliced?: number;
|
|
225
|
+
/** Unkeyed rows a virtualizer removed, restored into their list. */
|
|
226
|
+
restored?: number;
|
|
227
|
+
/**
|
|
228
|
+
* True when nothing could be restored on a page that reacted, so the walk
|
|
229
|
+
* scrolled back to where the page opened before handing it to the
|
|
230
|
+
* provider's live read.
|
|
231
|
+
*/
|
|
232
|
+
rewound?: boolean;
|
|
198
233
|
/**
|
|
199
234
|
* Whether the page reacted to scrolling (added/removed elements). False
|
|
200
235
|
* means the walk took the scroll-inert fast path to the bottom.
|
|
@@ -224,6 +259,7 @@ export interface DeepCaptureParams {
|
|
|
224
259
|
stepSettleMs: number;
|
|
225
260
|
bottomPollMs: number;
|
|
226
261
|
bottomPatienceMs: number;
|
|
262
|
+
inertPatienceMs: number;
|
|
227
263
|
inertSteps: number;
|
|
228
264
|
maxSteps: number;
|
|
229
265
|
stashMaxEntries: number;
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Block-level content classification — the `blk` candidate.
|
|
3
|
+
*
|
|
4
|
+
* Readability picks ONE container, so pages whose content is spread across
|
|
5
|
+
* many sections (service pages, product pages with spec blocks) lose recall,
|
|
6
|
+
* while the whole-document fallback keeps everything and loses precision.
|
|
7
|
+
* Here every text block (the text a block element owns directly) is scored
|
|
8
|
+
* independently by gradient-boosted trees (trained offline by
|
|
9
|
+
* `benchmark/train-blocks.py` on WCXB + ScrapingHub ground truth), and the
|
|
10
|
+
* kept blocks are re-emitted in document order — recall and precision are
|
|
11
|
+
* traded per block instead of per container.
|
|
12
|
+
*
|
|
13
|
+
* @module dsh-web-fetch-playwright/blocks
|
|
14
|
+
*/
|
|
15
|
+
import { type TreeEnsemble } from './trees.ts';
|
|
16
|
+
/**
|
|
17
|
+
* Word tokens: runs of letters/digits, except that each Han / kana character
|
|
18
|
+
* is its own token. Unspaced scripts would otherwise count a whole sentence
|
|
19
|
+
* as one "word", making a long Chinese paragraph weigh the same as a
|
|
20
|
+
* one-line headline link and breaking every density and ratio feature.
|
|
21
|
+
* Pages without CJK text tokenize exactly as with plain `[\p{L}\p{N}_]+`.
|
|
22
|
+
*/
|
|
23
|
+
export declare const WORD: RegExp;
|
|
24
|
+
/** One text block. */
|
|
25
|
+
export interface Block {
|
|
26
|
+
element: Element;
|
|
27
|
+
tag: string;
|
|
28
|
+
text: string;
|
|
29
|
+
words: number;
|
|
30
|
+
linkWords: number;
|
|
31
|
+
links: number;
|
|
32
|
+
depth: number;
|
|
33
|
+
negative: boolean;
|
|
34
|
+
positive: boolean;
|
|
35
|
+
inMain: boolean;
|
|
36
|
+
inRecord: boolean;
|
|
37
|
+
}
|
|
38
|
+
/** Normalized key used to match a block against another extraction's blocks. */
|
|
39
|
+
export declare function blockKey(text: string): string;
|
|
40
|
+
/**
|
|
41
|
+
* Split `root` into text blocks: each text node belongs to its nearest
|
|
42
|
+
* block-level ancestor, and a block is an element with directly owned text.
|
|
43
|
+
*/
|
|
44
|
+
export declare function segmentBlocks(root: Element, main?: Element | null, recordItems?: Set<Element>): Block[];
|
|
45
|
+
/** Block-level feature names (page features are prepended). */
|
|
46
|
+
export declare const BLOCK_FEATURES: readonly ["b_words_log", "b_chars_log", "b_link_density", "b_links_log", "b_p", "b_li", "b_h1", "b_hx", "b_td", "b_pre", "b_quote", "b_dd", "b_div", "b_depth", "b_pos", "b_in_art", "b_in_main", "b_in_record", "b_negative", "b_positive", "b_punct", "b_digits", "b_price", "b_prev_in_art", "b_next_in_art", "b_window_words_log", "b_window_link_density", "b_words_vs_max"];
|
|
47
|
+
/** Feature vectors for every block (page vector excluded). */
|
|
48
|
+
export declare function blockFeatures(blocks: Block[], artKeys: Set<string>): number[][];
|
|
49
|
+
/** A trained block classifier (log-odds GBDT). */
|
|
50
|
+
export interface BlockModel extends TreeEnsemble {
|
|
51
|
+
version: string;
|
|
52
|
+
threshold: number;
|
|
53
|
+
}
|
|
54
|
+
/** Override the block model (benchmark cross-fitting only). */
|
|
55
|
+
export declare function setBlockModel(model: BlockModel | null): void;
|
|
56
|
+
/** Whether a block model is available. */
|
|
57
|
+
export declare function hasBlockModel(): boolean;
|
|
58
|
+
/** Probability that each block is content. */
|
|
59
|
+
export declare function classifyBlocks(vectors: number[][]): number[];
|
|
60
|
+
/** Probability cut for keeping a block. */
|
|
61
|
+
export declare function blockThreshold(): number;
|
|
62
|
+
/**
|
|
63
|
+
* Re-emit kept blocks as HTML in document order: headings and `pre` keep
|
|
64
|
+
* their tag, list items are grouped into lists, everything else becomes a
|
|
65
|
+
* paragraph of the block's own inline content.
|
|
66
|
+
*/
|
|
67
|
+
export declare function renderBlocks(blocks: Block[], keep: boolean[]): string;
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Candidate generation for the denoise pipeline.
|
|
3
|
+
*
|
|
4
|
+
* The pipeline used to be ONE chain of hard decisions (Readability → coverage
|
|
5
|
+
* gate → trailing prune → whole-document fallback), each tuned by a global
|
|
6
|
+
* threshold. Every page shape that disagrees with a threshold's notion of
|
|
7
|
+
* "content" (link-dense docs indexes vs. recommendation rails, noscript
|
|
8
|
+
* server fallbacks on forums vs. lazy-image wrappers elsewhere) turned each
|
|
9
|
+
* fix into a regression somewhere else. Here every heuristic instead
|
|
10
|
+
* *proposes* a candidate extraction, and a learned selector
|
|
11
|
+
* (`./selector.ts`) picks one per page from page-level and candidate-level
|
|
12
|
+
* features — adding a heuristic can only add an option, never silently
|
|
13
|
+
* override the others.
|
|
14
|
+
*
|
|
15
|
+
* Candidates (see {@link CandidateKind}):
|
|
16
|
+
* - `art` Readability, trailing link farms pruned (the 0.2.18 article path)
|
|
17
|
+
* - `artu` Readability unpruned (link directories that ARE content)
|
|
18
|
+
* - `artx` Readability with a lenient link-density modifier
|
|
19
|
+
* - `main` the page's semantic main container (`<main>`, `[role=main]`, …)
|
|
20
|
+
* - `doc` the whole sanitized document (the 0.2.18 fallback)
|
|
21
|
+
* - `docc` the whole document minus consent/cookie banners, hidden and
|
|
22
|
+
* chrome-classed subtrees
|
|
23
|
+
* - `rec` the dominant repeated-record region (product cards, list rows)
|
|
24
|
+
* - `ns_art` / `ns_doc` the same two paths after unwrapping `<noscript>`
|
|
25
|
+
* server fallbacks (Discourse and friends ship their whole thread
|
|
26
|
+
* there for script-less clients)
|
|
27
|
+
* - `pre` Discourse `data-preloaded` post stream
|
|
28
|
+
* - `blk` blocks kept by the block classifier (`./blocks.ts`), added by
|
|
29
|
+
* the orchestrator once page features exist
|
|
30
|
+
*
|
|
31
|
+
* @module dsh-web-fetch-playwright/candidates
|
|
32
|
+
*/
|
|
33
|
+
/** Every candidate extraction strategy, in the selector's one-hot order. */
|
|
34
|
+
export declare const CANDIDATE_KINDS: readonly ["art", "artu", "artx", "main", "doc", "docc", "rec", "ns_art", "ns_doc", "pre", "blk"];
|
|
35
|
+
export type CandidateKind = typeof CANDIDATE_KINDS[number];
|
|
36
|
+
/** One proposed extraction, still as (unsanitized) HTML. */
|
|
37
|
+
export interface RawCandidate {
|
|
38
|
+
kind: CandidateKind;
|
|
39
|
+
/** HTML of the extraction; sanitized by the caller. */
|
|
40
|
+
html: string;
|
|
41
|
+
/** Title to prepend when the body does not open with a heading. */
|
|
42
|
+
title?: string;
|
|
43
|
+
/** Whether the 0.2.18 pipeline would have returned exactly this candidate. */
|
|
44
|
+
legacy: boolean;
|
|
45
|
+
}
|
|
46
|
+
/** Layout/noise tags DOMPurify removes outright (with their content). */
|
|
47
|
+
export declare const FORBID_TAGS: string[];
|
|
48
|
+
/** Tags whose subtrees never count as visible text. */
|
|
49
|
+
export declare const NOISE_SUBTREE_TAGS: Set<string>;
|
|
50
|
+
/**
|
|
51
|
+
* Count non-whitespace characters under `root`, skipping noise subtrees.
|
|
52
|
+
* Whitespace is ignored so CJK and latin pages measure comparable densities.
|
|
53
|
+
*/
|
|
54
|
+
export declare function visibleTextLength(root: Node | null, skip?: Set<string>): number;
|
|
55
|
+
/** Count non-whitespace characters under `root` that sit inside an anchor. */
|
|
56
|
+
export declare function linkTextLength(root: Node | null): number;
|
|
57
|
+
/** Minimum characters a trusted Readability extraction carries. */
|
|
58
|
+
export declare const GATE_MIN_ARTICLE_TEXT_CHARS = 250;
|
|
59
|
+
/**
|
|
60
|
+
* The 0.2.18 acceptance gate: distrust an extraction that is tiny in absolute
|
|
61
|
+
* terms or covers a sliver of the page's visible text. Kept as a selector
|
|
62
|
+
* feature and as the legacy-choice marker; it no longer decides alone.
|
|
63
|
+
*/
|
|
64
|
+
export declare function articleCoversPage(articleHtml: string, document: Document): boolean;
|
|
65
|
+
/** Prune with the 0.2.18 misfire guard; returns the original on a misfire. */
|
|
66
|
+
export declare function prunedArticleContent(articleHtml: string, document: Document): string;
|
|
67
|
+
/**
|
|
68
|
+
* A copy of `document` whose recommendation rails ("post_recommends",
|
|
69
|
+
* "related-posts", Outbrain/Taboola widgets) carry a marker attribute, or
|
|
70
|
+
* null when it has none. Readability scores class/id names and text, never
|
|
71
|
+
* `data-*`, so it picks the same container on the copy as on the original —
|
|
72
|
+
* removing the rails beforehand instead was measured to shift its pick to a
|
|
73
|
+
* larger ancestor (WCXB 4206).
|
|
74
|
+
*/
|
|
75
|
+
export declare function markRecommendationRails(document: Document): Document | null;
|
|
76
|
+
/**
|
|
77
|
+
* Split a Readability extraction of a rail-marked page into the extraction
|
|
78
|
+
* exactly as it would be unmarked, and — when a marked rail follows prose —
|
|
79
|
+
* the extraction without those rails.
|
|
80
|
+
*
|
|
81
|
+
* The trailing prune cannot catch a rail that outweighs the article (on a
|
|
82
|
+
* short 163.com post the rail is 86% of Readability's pick, past the share
|
|
83
|
+
* guards that keep docs indexes intact), and Readability strips the class
|
|
84
|
+
* names that would tell the two apart, hence the marker. A rail is removed
|
|
85
|
+
* only when it is recognised by name AND link density AND sits after some
|
|
86
|
+
* prose inside the extraction: a category page whose "related-product" grid
|
|
87
|
+
* opens the content (WCXB 4247) is the content, not a rail.
|
|
88
|
+
*/
|
|
89
|
+
export declare function splitRails(articleHtml: string, document: Document): {
|
|
90
|
+
plain: string;
|
|
91
|
+
derailed: string | null;
|
|
92
|
+
};
|
|
93
|
+
/**
|
|
94
|
+
* An element's element children as an array. Walks sibling pointers:
|
|
95
|
+
* jsdom's `HTMLCollection` is a proxy whose indexed access dominates
|
|
96
|
+
* profiles when `Array.from(el.children)` runs on every element of a large
|
|
97
|
+
* page (measured: 5.2s for a 4,000-paragraph article).
|
|
98
|
+
*/
|
|
99
|
+
export declare function childElements(element: Element): Element[];
|
|
100
|
+
/**
|
|
101
|
+
* Rewrite every `<noscript>` into a `<div>` holding its markup.
|
|
102
|
+
*
|
|
103
|
+
* Script-less parsing (jsdom, static snapshots) exposes noscript content as
|
|
104
|
+
* markup that DOMPurify and Readability then delete wholesale — on Discourse
|
|
105
|
+
* that is the entire server-rendered thread, measured 16–36KB per page.
|
|
106
|
+
* Scripting-enabled parsing leaves it as one raw text node, which is
|
|
107
|
+
* reparsed here so both parser modes converge.
|
|
108
|
+
*/
|
|
109
|
+
export declare function unwrapNoscripts(document: Document): number;
|
|
110
|
+
/** Visible text inside `<noscript>` elements (parsed either way). */
|
|
111
|
+
export declare function noscriptTextLength(document: Document): number;
|
|
112
|
+
/**
|
|
113
|
+
* Cookie/consent banner wording (EN/DE/FR/ES/IT/NL/ZH), as a regex source
|
|
114
|
+
* shared with the selector's `consent_hits` feature so both read the same
|
|
115
|
+
* languages. Chinese banners say "Cookie" or 隐私; 同意 ("agree") alone is
|
|
116
|
+
* everyday prose and only counts as the banner's ACTION below.
|
|
117
|
+
*/
|
|
118
|
+
export declare const CONSENT_TERMS: string;
|
|
119
|
+
/**
|
|
120
|
+
* Strip page chrome from a cloned body: chrome-named/roled subtrees, hidden
|
|
121
|
+
* subtrees, and consent banners recognised by their wording. A subtree is
|
|
122
|
+
* only removed while it is a minority of the page, so a site that names its
|
|
123
|
+
* article wrapper `content-modal` cannot lose everything.
|
|
124
|
+
*/
|
|
125
|
+
export declare function stripPageChrome(body: Element): void;
|
|
126
|
+
/** Record-region discovery result. */
|
|
127
|
+
export interface RecordRegion {
|
|
128
|
+
items: Element[];
|
|
129
|
+
/** Total visible text of the items. */
|
|
130
|
+
text: number;
|
|
131
|
+
}
|
|
132
|
+
/**
|
|
133
|
+
* Find the dominant repeated-record region: the sibling group with the same
|
|
134
|
+
* tag + class signature, at least four members, carrying the most text —
|
|
135
|
+
* product cards on a collection page, rows on a listing. Menu-like groups
|
|
136
|
+
* (tiny items, flat subtrees) are skipped.
|
|
137
|
+
*/
|
|
138
|
+
export declare function findRecordRegion(body: Element): RecordRegion | null;
|
|
139
|
+
/**
|
|
140
|
+
* Discourse ships the topic's post stream as JSON in
|
|
141
|
+
* `#data-preloaded[data-preloaded]` (a map of JSON strings). Rebuild the
|
|
142
|
+
* thread as HTML: author, date, cooked post body.
|
|
143
|
+
*/
|
|
144
|
+
export declare function discoursePreloadedHtml(document: Document): string | null;
|
|
145
|
+
/** Everything candidate generation learned about the page, for features. */
|
|
146
|
+
export interface GenerationFacts {
|
|
147
|
+
readabilityNull: boolean;
|
|
148
|
+
gatePass: boolean;
|
|
149
|
+
pruneChanged: boolean;
|
|
150
|
+
noscriptText: number;
|
|
151
|
+
record: RecordRegion | null;
|
|
152
|
+
hasMain: boolean;
|
|
153
|
+
hasPreloaded: boolean;
|
|
154
|
+
/** The semantic main container, when found. */
|
|
155
|
+
main: Element | null;
|
|
156
|
+
}
|
|
157
|
+
/**
|
|
158
|
+
* Propose every candidate extraction for one parsed page.
|
|
159
|
+
* @param document - the parsed page (data URIs already elided); not mutated.
|
|
160
|
+
*/
|
|
161
|
+
export declare function generateCandidates(document: Document): {
|
|
162
|
+
candidates: RawCandidate[];
|
|
163
|
+
facts: GenerationFacts;
|
|
164
|
+
};
|
|
@@ -37,16 +37,6 @@ export interface PlaywrightSettings {
|
|
|
37
37
|
denoise?: boolean;
|
|
38
38
|
/** How many fetches may render at once (1–200); blank = backend default. */
|
|
39
39
|
maxConcurrency?: number;
|
|
40
|
-
/**
|
|
41
|
-
* Bounded wait (ms) for a Cloudflare challenge to clear naturally in the
|
|
42
|
-
* same tab (0–60000); 0 = off (return the first response as-is).
|
|
43
|
-
*/
|
|
44
|
-
challengeWaitMs?: number;
|
|
45
|
-
/**
|
|
46
|
-
* Bounded budget (ms) for the deep-capture scroll walk (0–30000); 0 = off
|
|
47
|
-
* (capture the document as it rests, the legacy single-instant behavior).
|
|
48
|
-
*/
|
|
49
|
-
scrollBudgetMs?: number;
|
|
50
40
|
}
|
|
51
41
|
/** What the Playwright card renders. */
|
|
52
42
|
export interface PlaywrightCardState extends CardShell {
|
|
@@ -63,10 +53,6 @@ export interface PlaywrightCardState extends CardShell {
|
|
|
63
53
|
denoise: CardFieldState;
|
|
64
54
|
/** Concurrency input (draft decimal integer). */
|
|
65
55
|
maxConcurrency: CardFieldState;
|
|
66
|
-
/** Challenge wait input (draft decimal integer of milliseconds). */
|
|
67
|
-
challengeWaitMs: CardFieldState;
|
|
68
|
-
/** Scroll-walk budget input (draft decimal integer of milliseconds). */
|
|
69
|
-
scrollBudgetMs: CardFieldState;
|
|
70
56
|
}
|
|
71
57
|
/** The registration-side face the card's slot entry injects. */
|
|
72
58
|
export interface PlaywrightCardFace extends CardActions {
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* @module dsh-web-fetch-playwright/client/locales
|
|
6
6
|
*/
|
|
7
7
|
/** Locale keys this card renders. */
|
|
8
|
-
export type PlaywrightCardLocaleKey = 'description' | 'backendLabel' | 'backendLocal' | 'backendLocalHint' | 'backendCdp' | 'backendCdpHint' | 'playwrightPath' | 'playwrightPathHint' | 'playwrightPathPlaceholder' | 'cdpEndpoint' | 'cdpEndpointHint' | 'shareBrowserContext' | 'shareBrowserContextHint' | 'contextMode' | 'isolateBrowserContext' | 'isolateBrowserContextHint' | 'contextIsolationScope' | 'contextScopeSession' | 'contextScopeSessionHint' | 'contextScopeGlobal' | 'contextScopeGlobalHint' | 'denoise' | 'denoiseHint' | 'maxConcurrency' | 'maxConcurrencyHint' | 'maxConcurrencyPlaceholder' | '
|
|
8
|
+
export type PlaywrightCardLocaleKey = 'description' | 'backendLabel' | 'backendLocal' | 'backendLocalHint' | 'backendCdp' | 'backendCdpHint' | 'playwrightPath' | 'playwrightPathHint' | 'playwrightPathPlaceholder' | 'cdpEndpoint' | 'cdpEndpointHint' | 'shareBrowserContext' | 'shareBrowserContextHint' | 'contextMode' | 'isolateBrowserContext' | 'isolateBrowserContextHint' | 'contextIsolationScope' | 'contextScopeSession' | 'contextScopeSessionHint' | 'contextScopeGlobal' | 'contextScopeGlobalHint' | 'denoise' | 'denoiseHint' | 'maxConcurrency' | 'maxConcurrencyHint' | 'maxConcurrencyPlaceholder' | 'overridden' | 'reset' | 'readOnly' | 'save' | 'saving' | 'discard' | 'saveFailed' | 'invalidText';
|
|
9
9
|
/** This plugin's dictionary namespace, merged into the locale key map. */
|
|
10
10
|
declare module '@deepseek-ai/dsh-client-ui-slots' {
|
|
11
11
|
interface LocaleNamespaceMap {
|
package/lib/types/config.d.ts
CHANGED
|
@@ -15,16 +15,16 @@
|
|
|
15
15
|
import type { Volatile } from '@deepseek-ai/cordis';
|
|
16
16
|
import z from '@deepseek-ai/schemastery';
|
|
17
17
|
/**
|
|
18
|
-
* Default
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
18
|
+
* Default CAP (ms) on the deep-capture scroll walk (`src/autoscroll.ts`).
|
|
19
|
+
* It bounds only pages that keep reacting to scrolling — virtualized feeds
|
|
20
|
+
* and load-more streams — where 20s covers ~25kpx at the careful cadence
|
|
21
|
+
* (`SCROLL_STEP_MOTION_MS` + `SCROLL_STEP_SETTLE_MS` per ~0.7-viewport
|
|
22
|
+
* step). It is not what a typical page pays: a page that never reacts takes
|
|
23
|
+
* `SCROLL_INERT_STEPS` careful steps, one jump to the bottom and a
|
|
24
|
+
* `SCROLL_INERT_PATIENCE_MS` hold, and a page that does not scroll is
|
|
25
|
+
* skipped outright.
|
|
24
26
|
*/
|
|
25
27
|
export declare const DEFAULT_SCROLL_BUDGET_MS = 20000;
|
|
26
|
-
/** Ceiling the schema accepts for `scrollBudgetMs`. */
|
|
27
|
-
export declare const MAX_SCROLL_BUDGET_MS = 30000;
|
|
28
28
|
/** Default CDP endpoint when the settings section leaves it blank. */
|
|
29
29
|
export declare const DEFAULT_CDP_ENDPOINT = "127.0.0.1:9222";
|
|
30
30
|
/**
|
|
@@ -45,16 +45,24 @@ export declare const MAX_CONCURRENCY_CEILING = 200;
|
|
|
45
45
|
/**
|
|
46
46
|
* Default bounded wait (ms) for a Cloudflare challenge to clear naturally —
|
|
47
47
|
* the user's real browser passes the verification on its own while the fetch
|
|
48
|
-
* holds the same page and context.
|
|
49
|
-
*
|
|
48
|
+
* holds the same page and context. Bounded further by the 25s per-fetch
|
|
49
|
+
* waiting budget, which keeps a settle/decode tail after the wait.
|
|
50
50
|
*/
|
|
51
51
|
export declare const DEFAULT_CHALLENGE_WAIT_MS = 15000;
|
|
52
|
-
/** Ceiling the schema accepts for `challengeWaitMs`. */
|
|
53
|
-
export declare const MAX_CHALLENGE_WAIT_MS = 60000;
|
|
54
52
|
/** Default same-page re-navigation attempts after a challenge wait runs out. */
|
|
55
53
|
export declare const DEFAULT_CHALLENGE_RETRIES = 1;
|
|
56
54
|
/** Ceiling the schema accepts for `challengeRetries`. */
|
|
57
55
|
export declare const MAX_CHALLENGE_RETRIES = 3;
|
|
56
|
+
/**
|
|
57
|
+
* Default for `challengeHttpFallback`: when the browser cannot clear a
|
|
58
|
+
* challenge inside its window (the interactive checkbox was never offered,
|
|
59
|
+
* or verification refused), refetch the URL once through the page's
|
|
60
|
+
* plain-HTTP request context — the same cookie jar, no render decision —
|
|
61
|
+
* because many Cloudflare edges serve the real document to non-browser
|
|
62
|
+
* clients while withholding the interactive challenge from the browser
|
|
63
|
+
* session (measured live: v2ex.com from a datacenter IP).
|
|
64
|
+
*/
|
|
65
|
+
export declare const DEFAULT_CHALLENGE_HTTP_FALLBACK = true;
|
|
58
66
|
/** Which browser backend serves a fetch. */
|
|
59
67
|
export type PlaywrightBackend = 'local' | 'cdp';
|
|
60
68
|
/**
|
|
@@ -98,7 +106,9 @@ export interface Config {
|
|
|
98
106
|
* Bounded wait (ms) for a Cloudflare challenge to clear naturally inside
|
|
99
107
|
* the same page/context. `0` (or any config leaving this at 0) restores the
|
|
100
108
|
* legacy behavior: the first response is final, no waiting — the A/B
|
|
101
|
-
* baseline.
|
|
109
|
+
* baseline. Code-level only (tests, demo scripts): not part of the
|
|
110
|
+
* settings schema, so users cannot configure it. Default:
|
|
111
|
+
* {@link DEFAULT_CHALLENGE_WAIT_MS}.
|
|
102
112
|
*/
|
|
103
113
|
challengeWaitMs?: number;
|
|
104
114
|
/**
|
|
@@ -107,13 +117,22 @@ export interface Config {
|
|
|
107
117
|
* Schema default: {@link DEFAULT_CHALLENGE_RETRIES}.
|
|
108
118
|
*/
|
|
109
119
|
challengeRetries?: number;
|
|
120
|
+
/**
|
|
121
|
+
* When the browser cannot clear a challenge inside its window (no
|
|
122
|
+
* checkbox offered, or verification refused), refetch the URL once
|
|
123
|
+
* through the page's plain-HTTP request context (same cookie jar) and
|
|
124
|
+
* return that document when it is not itself a challenge. `false`
|
|
125
|
+
* restores the browser-only posture.
|
|
126
|
+
*/
|
|
127
|
+
challengeHttpFallback?: boolean;
|
|
110
128
|
/**
|
|
111
129
|
* Bounded budget (ms) for the deep-capture scroll walk — the in-page pass
|
|
112
130
|
* that defeats viewport virtualization on feed-shaped pages (Discourse
|
|
113
131
|
* topic streams: replies outside the render window are empty height-only
|
|
114
132
|
* placeholders carrying the post's id; see `src/autoscroll.ts`). `0`
|
|
115
|
-
* restores the legacy single-instant capture.
|
|
116
|
-
*
|
|
133
|
+
* restores the legacy single-instant capture. Code-level only (tests,
|
|
134
|
+
* demo scripts): not part of the settings schema, so users cannot
|
|
135
|
+
* configure it. Default: {@link DEFAULT_SCROLL_BUDGET_MS}.
|
|
117
136
|
*/
|
|
118
137
|
scrollBudgetMs?: number;
|
|
119
138
|
/**
|
|
@@ -131,9 +150,8 @@ export declare const Config: z<Schemastery.ObjectS<NoInfer<{
|
|
|
131
150
|
contextIsolationScope: z<"session" | "global", "session" | "global", "volatile-defined">;
|
|
132
151
|
denoise: z<boolean, boolean, "volatile-defined">;
|
|
133
152
|
maxConcurrency: z<number, number, "volatile">;
|
|
134
|
-
challengeWaitMs: z<number, number, "volatile-defined">;
|
|
135
153
|
challengeRetries: z<number, number, "volatile-defined">;
|
|
136
|
-
|
|
154
|
+
challengeHttpFallback: z<boolean, boolean, "volatile-defined">;
|
|
137
155
|
}>>, Schemastery.ObjectT<NoInfer<{
|
|
138
156
|
backend: z<"local" | "cdp", "local" | "cdp", "volatile-defined">;
|
|
139
157
|
playwrightPath: z<string, string, "volatile-defined">;
|
|
@@ -142,9 +160,8 @@ export declare const Config: z<Schemastery.ObjectS<NoInfer<{
|
|
|
142
160
|
contextIsolationScope: z<"session" | "global", "session" | "global", "volatile-defined">;
|
|
143
161
|
denoise: z<boolean, boolean, "volatile-defined">;
|
|
144
162
|
maxConcurrency: z<number, number, "volatile">;
|
|
145
|
-
challengeWaitMs: z<number, number, "volatile-defined">;
|
|
146
163
|
challengeRetries: z<number, number, "volatile-defined">;
|
|
147
|
-
|
|
164
|
+
challengeHttpFallback: z<boolean, boolean, "volatile-defined">;
|
|
148
165
|
}>>, "plain">;
|
|
149
166
|
/**
|
|
150
167
|
* The config `apply` receives once the schema resolves: every volatile field
|
|
@@ -169,12 +186,10 @@ export interface PlaywrightPluginConfig {
|
|
|
169
186
|
* field owns no default — the backend resolves one).
|
|
170
187
|
*/
|
|
171
188
|
maxConcurrency: Volatile<number | undefined>;
|
|
172
|
-
/** Challenge wait budget (ms) as a live reference. */
|
|
173
|
-
challengeWaitMs: Volatile<number>;
|
|
174
189
|
/** Challenge retry count as a live reference. */
|
|
175
190
|
challengeRetries: Volatile<number>;
|
|
176
|
-
/**
|
|
177
|
-
|
|
191
|
+
/** Challenge dead-end plain-HTTP refetch toggle as a live reference. */
|
|
192
|
+
challengeHttpFallback: Volatile<boolean>;
|
|
178
193
|
}
|
|
179
194
|
/**
|
|
180
195
|
* Read the live references `apply` holds into one plain input for
|
|
@@ -198,13 +213,14 @@ export declare function snapshotsOf(config: PlaywrightPluginConfig): Config;
|
|
|
198
213
|
export declare function resolveConfig(config: Partial<Config>): ResolvedConfig;
|
|
199
214
|
/**
|
|
200
215
|
* Complete config after schemastery applies the field defaults it owns.
|
|
201
|
-
* `maxConcurrency`
|
|
202
|
-
*
|
|
203
|
-
*
|
|
204
|
-
*
|
|
216
|
+
* `maxConcurrency` stays optional because its effective default depends on
|
|
217
|
+
* `backend`; `challengeWaitMs` and `scrollBudgetMs` stay optional because
|
|
218
|
+
* they are code-level overrides outside the schema — absent, the
|
|
219
|
+
* effective* helpers supply the built-in defaults.
|
|
205
220
|
*/
|
|
206
|
-
export type ResolvedConfig = Omit<Required<Config>, 'maxConcurrency' | 'scrollBudgetMs'> & {
|
|
221
|
+
export type ResolvedConfig = Omit<Required<Config>, 'maxConcurrency' | 'challengeWaitMs' | 'scrollBudgetMs'> & {
|
|
207
222
|
maxConcurrency?: number;
|
|
223
|
+
challengeWaitMs?: number;
|
|
208
224
|
scrollBudgetMs?: number;
|
|
209
225
|
};
|
|
210
226
|
/**
|
|
@@ -218,7 +234,7 @@ export type ResolvedConfig = Omit<Required<Config>, 'maxConcurrency' | 'scrollBu
|
|
|
218
234
|
export declare function effectiveMaxConcurrency(config: Pick<Config, 'backend' | 'maxConcurrency'>): number;
|
|
219
235
|
/**
|
|
220
236
|
* The challenge wait budget a fetch actually runs with: an explicit setting
|
|
221
|
-
* wins, else the
|
|
237
|
+
* wins, else the built-in default. `0` means the challenge path is off.
|
|
222
238
|
*
|
|
223
239
|
* @param config - the resolved settings section (or any partial of it).
|
|
224
240
|
* @returns the bounded wait in milliseconds.
|
|
@@ -232,9 +248,17 @@ export declare function effectiveChallengeWaitMs(config: Pick<Config, 'challenge
|
|
|
232
248
|
* @returns retries after a wait window runs out.
|
|
233
249
|
*/
|
|
234
250
|
export declare function effectiveChallengeRetries(config: Pick<Config, 'challengeRetries'>): number;
|
|
251
|
+
/**
|
|
252
|
+
* Whether a challenge dead-end may refetch through plain HTTP: an explicit
|
|
253
|
+
* setting wins (absent = on, the default).
|
|
254
|
+
*
|
|
255
|
+
* @param config - the resolved settings section (or any partial of it).
|
|
256
|
+
* @returns true when the plain-HTTP fallback may run.
|
|
257
|
+
*/
|
|
258
|
+
export declare function effectiveChallengeHttpFallback(config: Pick<Config, 'challengeHttpFallback'>): boolean;
|
|
235
259
|
/**
|
|
236
260
|
* The scroll-walk budget a fetch actually runs with: an explicit setting
|
|
237
|
-
* wins, else the
|
|
261
|
+
* wins, else the built-in default. `0` means the deep-capture walk is off.
|
|
238
262
|
*
|
|
239
263
|
* @param config - the resolved settings section (or any partial of it).
|
|
240
264
|
* @returns the bounded walk budget in milliseconds.
|
package/lib/types/index.d.ts
CHANGED
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
*/
|
|
16
16
|
import type { Context } from '@deepseek-ai/cordis';
|
|
17
17
|
import type { PlaywrightPluginConfig } from './config.ts';
|
|
18
|
-
export { Config, DEFAULT_CDP_ENDPOINT, DEFAULT_CHALLENGE_RETRIES, DEFAULT_CHALLENGE_WAIT_MS, DEFAULT_SCROLL_BUDGET_MS,
|
|
18
|
+
export { Config, DEFAULT_CDP_ENDPOINT, DEFAULT_CHALLENGE_HTTP_FALLBACK, DEFAULT_CHALLENGE_RETRIES, DEFAULT_CHALLENGE_WAIT_MS, DEFAULT_SCROLL_BUDGET_MS, effectiveChallengeHttpFallback, effectiveChallengeRetries, effectiveChallengeWaitMs, effectiveContextMode, effectiveScrollBudgetMs, normalizeCdpEndpoint, resolveConfig, snapshotsOf, } from './config.ts';
|
|
19
19
|
export type { Config as PlaywrightFetchConfig, CdpContextMode, ContextIsolationScope, PlaywrightBackend, PlaywrightPluginConfig, ResolvedConfig, } from './config.ts';
|
|
20
20
|
export { CdpConnectionPool } from './cdp-pool.ts';
|
|
21
21
|
export type { CdpAcquireMode, CdpConnect, CdpLease } from './cdp-pool.ts';
|
|
@@ -24,10 +24,12 @@ export type { ChallengeVerdict } from './challenge.ts';
|
|
|
24
24
|
export { deepCaptureWalk } from './autoscroll.ts';
|
|
25
25
|
export type { DeepCapture, DeepCaptureStats } from './autoscroll.ts';
|
|
26
26
|
export { PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider, WEB_FETCH_CHALLENGE_CODE } from './provider.ts';
|
|
27
|
-
export { htmlToMarkdown } from './markdown.ts';
|
|
27
|
+
export { blockTrainingRows, htmlToMarkdown, renderCandidate, scoreCandidates } from './markdown.ts';
|
|
28
|
+
export { BLOCK_FEATURES, setBlockModel } from './blocks.ts';
|
|
28
29
|
export { extractPdfMarkdown, isPdfBytes, isPdfContentType, layoutPdfMarkdown, mayBePdf, MAX_PDF_BYTES, MAX_PDF_PAGES, readPdfDocument, WEB_PDF_ENCRYPTED_CODE, WEB_PDF_INVALID_CODE, WEB_PDF_NO_TEXT_CODE, WEB_PDF_TOO_LARGE_CODE } from './pdf.ts';
|
|
29
30
|
export type { PdfDocumentText, PdfExtraction, PdfExtractOptions, PdfOutlineEntry, PdfPageText, PdfTextRun } from './pdf.ts';
|
|
30
|
-
export type { DenoiseMode, DenoiseResult } from './markdown.ts';
|
|
31
|
+
export type { DenoiseMode, DenoiseResult, ScoredCandidate } from './markdown.ts';
|
|
32
|
+
export { FEATURE_NAMES, PAGE_FEATURES } from './selector.ts';
|
|
31
33
|
/** Cordis plugin name used by loader diagnostics. */
|
|
32
34
|
export declare const name = "dsh-web-fetch-playwright";
|
|
33
35
|
/**
|