dsh-web-fetch-playwright 0.2.19 → 0.2.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/client.js +16 -16
- package/lib/index.js +57160 -379
- package/lib/types/block-model.d.ts +2 -0
- package/lib/types/blocks.d.ts +67 -0
- package/lib/types/candidates.d.ts +157 -0
- package/lib/types/index.d.ts +4 -2
- package/lib/types/markdown.d.ts +50 -24
- package/lib/types/selector-model.d.ts +2 -0
- package/lib/types/selector.d.ts +58 -0
- package/lib/types/trees.d.ts +25 -0
- package/package.json +1 -1
- package/src/block-model.ts +5 -0
- package/src/blocks.ts +223 -0
- package/src/candidates.ts +639 -0
- package/src/index.ts +4 -2
- package/src/markdown.ts +136 -413
- package/src/selector-model.ts +6 -0
- package/src/selector.ts +277 -0
- package/src/trees.ts +43 -0
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Block-level content classification — the `blk` candidate.
|
|
3
|
+
*
|
|
4
|
+
* Readability picks ONE container, so pages whose content is spread across
|
|
5
|
+
* many sections (service pages, product pages with spec blocks) lose recall,
|
|
6
|
+
* while the whole-document fallback keeps everything and loses precision.
|
|
7
|
+
* Here every text block (the text a block element owns directly) is scored
|
|
8
|
+
* independently by gradient-boosted trees (trained offline by
|
|
9
|
+
* `benchmark/train-blocks.py` on WCXB + ScrapingHub ground truth), and the
|
|
10
|
+
* kept blocks are re-emitted in document order — recall and precision are
|
|
11
|
+
* traded per block instead of per container.
|
|
12
|
+
*
|
|
13
|
+
* @module dsh-web-fetch-playwright/blocks
|
|
14
|
+
*/
|
|
15
|
+
import { type TreeEnsemble } from './trees.ts';
|
|
16
|
+
/**
|
|
17
|
+
* Word tokens: runs of letters/digits, except that each Han / kana character
|
|
18
|
+
* is its own token. Unspaced scripts would otherwise count a whole sentence
|
|
19
|
+
* as one "word", making a long Chinese paragraph weigh the same as a
|
|
20
|
+
* one-line headline link and breaking every density and ratio feature.
|
|
21
|
+
* Pages without CJK text tokenize exactly as with plain `[\p{L}\p{N}_]+`.
|
|
22
|
+
*/
|
|
23
|
+
export declare const WORD: RegExp;
|
|
24
|
+
/** One text block. */
|
|
25
|
+
export interface Block {
|
|
26
|
+
element: Element;
|
|
27
|
+
tag: string;
|
|
28
|
+
text: string;
|
|
29
|
+
words: number;
|
|
30
|
+
linkWords: number;
|
|
31
|
+
links: number;
|
|
32
|
+
depth: number;
|
|
33
|
+
negative: boolean;
|
|
34
|
+
positive: boolean;
|
|
35
|
+
inMain: boolean;
|
|
36
|
+
inRecord: boolean;
|
|
37
|
+
}
|
|
38
|
+
/** Normalized key used to match a block against another extraction's blocks. */
|
|
39
|
+
export declare function blockKey(text: string): string;
|
|
40
|
+
/**
|
|
41
|
+
* Split `root` into text blocks: each text node belongs to its nearest
|
|
42
|
+
* block-level ancestor, and a block is an element with directly owned text.
|
|
43
|
+
*/
|
|
44
|
+
export declare function segmentBlocks(root: Element, main?: Element | null, recordItems?: Set<Element>): Block[];
|
|
45
|
+
/** Block-level feature names (page features are prepended). */
|
|
46
|
+
export declare const BLOCK_FEATURES: readonly ["b_words_log", "b_chars_log", "b_link_density", "b_links_log", "b_p", "b_li", "b_h1", "b_hx", "b_td", "b_pre", "b_quote", "b_dd", "b_div", "b_depth", "b_pos", "b_in_art", "b_in_main", "b_in_record", "b_negative", "b_positive", "b_punct", "b_digits", "b_price", "b_prev_in_art", "b_next_in_art", "b_window_words_log", "b_window_link_density", "b_words_vs_max"];
|
|
47
|
+
/** Feature vectors for every block (page vector excluded). */
|
|
48
|
+
export declare function blockFeatures(blocks: Block[], artKeys: Set<string>): number[][];
|
|
49
|
+
/** A trained block classifier (log-odds GBDT). */
|
|
50
|
+
export interface BlockModel extends TreeEnsemble {
|
|
51
|
+
version: string;
|
|
52
|
+
threshold: number;
|
|
53
|
+
}
|
|
54
|
+
/** Override the block model (benchmark cross-fitting only). */
|
|
55
|
+
export declare function setBlockModel(model: BlockModel | null): void;
|
|
56
|
+
/** Whether a block model is available. */
|
|
57
|
+
export declare function hasBlockModel(): boolean;
|
|
58
|
+
/** Probability that each block is content. */
|
|
59
|
+
export declare function classifyBlocks(vectors: number[][]): number[];
|
|
60
|
+
/** Probability cut for keeping a block. */
|
|
61
|
+
export declare function blockThreshold(): number;
|
|
62
|
+
/**
|
|
63
|
+
* Re-emit kept blocks as HTML in document order: headings and `pre` keep
|
|
64
|
+
* their tag, list items are grouped into lists, everything else becomes a
|
|
65
|
+
* paragraph of the block's own inline content.
|
|
66
|
+
*/
|
|
67
|
+
export declare function renderBlocks(blocks: Block[], keep: boolean[]): string;
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Candidate generation for the denoise pipeline.
|
|
3
|
+
*
|
|
4
|
+
* The pipeline used to be ONE chain of hard decisions (Readability → coverage
|
|
5
|
+
* gate → trailing prune → whole-document fallback), each tuned by a global
|
|
6
|
+
* threshold. Every page shape that disagrees with a threshold's notion of
|
|
7
|
+
* "content" (link-dense docs indexes vs. recommendation rails, noscript
|
|
8
|
+
* server fallbacks on forums vs. lazy-image wrappers elsewhere) turned each
|
|
9
|
+
* fix into a regression somewhere else. Here every heuristic instead
|
|
10
|
+
* *proposes* a candidate extraction, and a learned selector
|
|
11
|
+
* (`./selector.ts`) picks one per page from page-level and candidate-level
|
|
12
|
+
* features — adding a heuristic can only add an option, never silently
|
|
13
|
+
* override the others.
|
|
14
|
+
*
|
|
15
|
+
* Candidates (see {@link CandidateKind}):
|
|
16
|
+
* - `art` Readability, trailing link farms pruned (the 0.2.18 article path)
|
|
17
|
+
* - `artu` Readability unpruned (link directories that ARE content)
|
|
18
|
+
* - `artx` Readability with a lenient link-density modifier
|
|
19
|
+
* - `main` the page's semantic main container (`<main>`, `[role=main]`, …)
|
|
20
|
+
* - `doc` the whole sanitized document (the 0.2.18 fallback)
|
|
21
|
+
* - `docc` the whole document minus consent/cookie banners, hidden and
|
|
22
|
+
* chrome-classed subtrees
|
|
23
|
+
* - `rec` the dominant repeated-record region (product cards, list rows)
|
|
24
|
+
* - `ns_art` / `ns_doc` the same two paths after unwrapping `<noscript>`
|
|
25
|
+
* server fallbacks (Discourse and friends ship their whole thread
|
|
26
|
+
* there for script-less clients)
|
|
27
|
+
* - `pre` Discourse `data-preloaded` post stream
|
|
28
|
+
* - `blk` blocks kept by the block classifier (`./blocks.ts`), added by
|
|
29
|
+
* the orchestrator once page features exist
|
|
30
|
+
*
|
|
31
|
+
* @module dsh-web-fetch-playwright/candidates
|
|
32
|
+
*/
|
|
33
|
+
/** Every candidate extraction strategy, in the selector's one-hot order. */
|
|
34
|
+
export declare const CANDIDATE_KINDS: readonly ["art", "artu", "artx", "main", "doc", "docc", "rec", "ns_art", "ns_doc", "pre", "blk"];
|
|
35
|
+
export type CandidateKind = typeof CANDIDATE_KINDS[number];
|
|
36
|
+
/** One proposed extraction, still as (unsanitized) HTML. */
|
|
37
|
+
export interface RawCandidate {
|
|
38
|
+
kind: CandidateKind;
|
|
39
|
+
/** HTML of the extraction; sanitized by the caller. */
|
|
40
|
+
html: string;
|
|
41
|
+
/** Title to prepend when the body does not open with a heading. */
|
|
42
|
+
title?: string;
|
|
43
|
+
/** Whether the 0.2.18 pipeline would have returned exactly this candidate. */
|
|
44
|
+
legacy: boolean;
|
|
45
|
+
}
|
|
46
|
+
/** Layout/noise tags DOMPurify removes outright (with their content). */
|
|
47
|
+
export declare const FORBID_TAGS: string[];
|
|
48
|
+
/** Tags whose subtrees never count as visible text. */
|
|
49
|
+
export declare const NOISE_SUBTREE_TAGS: Set<string>;
|
|
50
|
+
/**
|
|
51
|
+
* Count non-whitespace characters under `root`, skipping noise subtrees.
|
|
52
|
+
* Whitespace is ignored so CJK and latin pages measure comparable densities.
|
|
53
|
+
*/
|
|
54
|
+
export declare function visibleTextLength(root: Node | null, skip?: Set<string>): number;
|
|
55
|
+
/** Count non-whitespace characters under `root` that sit inside an anchor. */
|
|
56
|
+
export declare function linkTextLength(root: Node | null): number;
|
|
57
|
+
/** Minimum characters a trusted Readability extraction carries. */
|
|
58
|
+
export declare const GATE_MIN_ARTICLE_TEXT_CHARS = 250;
|
|
59
|
+
/**
|
|
60
|
+
* The 0.2.18 acceptance gate: distrust an extraction that is tiny in absolute
|
|
61
|
+
* terms or covers a sliver of the page's visible text. Kept as a selector
|
|
62
|
+
* feature and as the legacy-choice marker; it no longer decides alone.
|
|
63
|
+
*/
|
|
64
|
+
export declare function articleCoversPage(articleHtml: string, document: Document): boolean;
|
|
65
|
+
/** Prune with the 0.2.18 misfire guard; returns the original on a misfire. */
|
|
66
|
+
export declare function prunedArticleContent(articleHtml: string, document: Document): string;
|
|
67
|
+
/**
|
|
68
|
+
* A copy of `document` whose recommendation rails ("post_recommends",
|
|
69
|
+
* "related-posts", Outbrain/Taboola widgets) carry a marker attribute, or
|
|
70
|
+
* null when it has none. Readability scores class/id names and text, never
|
|
71
|
+
* `data-*`, so it picks the same container on the copy as on the original —
|
|
72
|
+
* removing the rails beforehand instead was measured to shift its pick to a
|
|
73
|
+
* larger ancestor (WCXB 4206).
|
|
74
|
+
*/
|
|
75
|
+
export declare function markRecommendationRails(document: Document): Document | null;
|
|
76
|
+
/**
|
|
77
|
+
* Split a Readability extraction of a rail-marked page into the extraction
|
|
78
|
+
* exactly as it would be unmarked, and — when a marked rail follows prose —
|
|
79
|
+
* the extraction without those rails.
|
|
80
|
+
*
|
|
81
|
+
* The trailing prune cannot catch a rail that outweighs the article (on a
|
|
82
|
+
* short 163.com post the rail is 86% of Readability's pick, past the share
|
|
83
|
+
* guards that keep docs indexes intact), and Readability strips the class
|
|
84
|
+
* names that would tell the two apart, hence the marker. A rail is removed
|
|
85
|
+
* only when it is recognised by name AND link density AND sits after some
|
|
86
|
+
* prose inside the extraction: a category page whose "related-product" grid
|
|
87
|
+
* opens the content (WCXB 4247) is the content, not a rail.
|
|
88
|
+
*/
|
|
89
|
+
export declare function splitRails(articleHtml: string, document: Document): {
|
|
90
|
+
plain: string;
|
|
91
|
+
derailed: string | null;
|
|
92
|
+
};
|
|
93
|
+
/**
|
|
94
|
+
* An element's element children as an array. Walks sibling pointers:
|
|
95
|
+
* jsdom's `HTMLCollection` is a proxy whose indexed access dominates
|
|
96
|
+
* profiles when `Array.from(el.children)` runs on every element of a large
|
|
97
|
+
* page (measured: 5.2s for a 4,000-paragraph article).
|
|
98
|
+
*/
|
|
99
|
+
export declare function childElements(element: Element): Element[];
|
|
100
|
+
/**
|
|
101
|
+
* Rewrite every `<noscript>` into a `<div>` holding its markup.
|
|
102
|
+
*
|
|
103
|
+
* Script-less parsing (jsdom, static snapshots) exposes noscript content as
|
|
104
|
+
* markup that DOMPurify and Readability then delete wholesale — on Discourse
|
|
105
|
+
* that is the entire server-rendered thread, measured 16–36KB per page.
|
|
106
|
+
* Scripting-enabled parsing leaves it as one raw text node, which is
|
|
107
|
+
* reparsed here so both parser modes converge.
|
|
108
|
+
*/
|
|
109
|
+
export declare function unwrapNoscripts(document: Document): number;
|
|
110
|
+
/** Visible text inside `<noscript>` elements (parsed either way). */
|
|
111
|
+
export declare function noscriptTextLength(document: Document): number;
|
|
112
|
+
/**
|
|
113
|
+
* Strip page chrome from a cloned body: chrome-named/roled subtrees, hidden
|
|
114
|
+
* subtrees, and consent banners recognised by their wording. A subtree is
|
|
115
|
+
* only removed while it is a minority of the page, so a site that names its
|
|
116
|
+
* article wrapper `content-modal` cannot lose everything.
|
|
117
|
+
*/
|
|
118
|
+
export declare function stripPageChrome(body: Element): void;
|
|
119
|
+
/** Record-region discovery result. */
|
|
120
|
+
export interface RecordRegion {
|
|
121
|
+
items: Element[];
|
|
122
|
+
/** Total visible text of the items. */
|
|
123
|
+
text: number;
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* Find the dominant repeated-record region: the sibling group with the same
|
|
127
|
+
* tag + class signature, at least four members, carrying the most text —
|
|
128
|
+
* product cards on a collection page, rows on a listing. Menu-like groups
|
|
129
|
+
* (tiny items, flat subtrees) are skipped.
|
|
130
|
+
*/
|
|
131
|
+
export declare function findRecordRegion(body: Element): RecordRegion | null;
|
|
132
|
+
/**
|
|
133
|
+
* Discourse ships the topic's post stream as JSON in
|
|
134
|
+
* `#data-preloaded[data-preloaded]` (a map of JSON strings). Rebuild the
|
|
135
|
+
* thread as HTML: author, date, cooked post body.
|
|
136
|
+
*/
|
|
137
|
+
export declare function discoursePreloadedHtml(document: Document): string | null;
|
|
138
|
+
/** Everything candidate generation learned about the page, for features. */
|
|
139
|
+
export interface GenerationFacts {
|
|
140
|
+
readabilityNull: boolean;
|
|
141
|
+
gatePass: boolean;
|
|
142
|
+
pruneChanged: boolean;
|
|
143
|
+
noscriptText: number;
|
|
144
|
+
record: RecordRegion | null;
|
|
145
|
+
hasMain: boolean;
|
|
146
|
+
hasPreloaded: boolean;
|
|
147
|
+
/** The semantic main container, when found. */
|
|
148
|
+
main: Element | null;
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* Propose every candidate extraction for one parsed page.
|
|
152
|
+
* @param document - the parsed page (data URIs already elided); not mutated.
|
|
153
|
+
*/
|
|
154
|
+
export declare function generateCandidates(document: Document): {
|
|
155
|
+
candidates: RawCandidate[];
|
|
156
|
+
facts: GenerationFacts;
|
|
157
|
+
};
|
package/lib/types/index.d.ts
CHANGED
|
@@ -24,10 +24,12 @@ export type { ChallengeVerdict } from './challenge.ts';
|
|
|
24
24
|
export { deepCaptureWalk } from './autoscroll.ts';
|
|
25
25
|
export type { DeepCapture, DeepCaptureStats } from './autoscroll.ts';
|
|
26
26
|
export { PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider, WEB_FETCH_CHALLENGE_CODE } from './provider.ts';
|
|
27
|
-
export { htmlToMarkdown } from './markdown.ts';
|
|
27
|
+
export { blockTrainingRows, htmlToMarkdown, renderCandidate, scoreCandidates } from './markdown.ts';
|
|
28
|
+
export { BLOCK_FEATURES, setBlockModel } from './blocks.ts';
|
|
28
29
|
export { extractPdfMarkdown, isPdfBytes, isPdfContentType, layoutPdfMarkdown, mayBePdf, MAX_PDF_BYTES, MAX_PDF_PAGES, readPdfDocument, WEB_PDF_ENCRYPTED_CODE, WEB_PDF_INVALID_CODE, WEB_PDF_NO_TEXT_CODE, WEB_PDF_TOO_LARGE_CODE } from './pdf.ts';
|
|
29
30
|
export type { PdfDocumentText, PdfExtraction, PdfExtractOptions, PdfOutlineEntry, PdfPageText, PdfTextRun } from './pdf.ts';
|
|
30
|
-
export type { DenoiseMode, DenoiseResult } from './markdown.ts';
|
|
31
|
+
export type { DenoiseMode, DenoiseResult, ScoredCandidate } from './markdown.ts';
|
|
32
|
+
export { FEATURE_NAMES, PAGE_FEATURES } from './selector.ts';
|
|
31
33
|
/** Cordis plugin name used by loader diagnostics. */
|
|
32
34
|
export declare const name = "dsh-web-fetch-playwright";
|
|
33
35
|
/**
|
package/lib/types/markdown.d.ts
CHANGED
|
@@ -1,21 +1,19 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* The denoise pipeline: rendered HTML → sanitized article HTML → markdown.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
* inline `data:` image payloads are elided to size placeholders
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
* options and span-safe table rules as the shipped `dsh-tool-web` renderer,
|
|
18
|
-
* so output is consistent with what `web_fetch` produces elsewhere.
|
|
4
|
+
* Since 0.2.20 the pipeline is "propose, then select": jsdom parses the
|
|
5
|
+
* page; inline `data:` image payloads are elided to size placeholders;
|
|
6
|
+
* every extraction heuristic proposes a candidate (`./candidates.ts` —
|
|
7
|
+
* Readability pruned/unpruned/lenient, the semantic main container, the
|
|
8
|
+
* whole document with and without consent/chrome stripping, the repeated
|
|
9
|
+
* record region, noscript-unwrapped variants, the Discourse preloaded post
|
|
10
|
+
* stream) plus the block classifier's pick (`./blocks.ts`); DOMPurify
|
|
11
|
+
* sanitizes each and forbids the layout tags noise lives in; a learned
|
|
12
|
+
* selector (`./selector.ts`) estimates each candidate's quality from page-
|
|
13
|
+
* and candidate-level features and the best one wins, with the 0.2.18
|
|
14
|
+
* choice kept unless another candidate clearly beats it. Turndown with the
|
|
15
|
+
* GFM plugin converts the winner using the same style options and span-safe
|
|
16
|
+
* table rules as the shipped `dsh-tool-web` renderer.
|
|
19
17
|
*
|
|
20
18
|
* Synchronous for its callers and unit-tested against fixture pages: jsdom
|
|
21
19
|
* itself is loaded lazily on the first call — and, inside a dsh profile, only
|
|
@@ -24,6 +22,8 @@
|
|
|
24
22
|
*
|
|
25
23
|
* @module dsh-web-fetch-playwright/markdown
|
|
26
24
|
*/
|
|
25
|
+
import { type CandidateKind } from './candidates.ts';
|
|
26
|
+
import { type CandidateStats } from './selector.ts';
|
|
27
27
|
/** Which extraction path produced a result. */
|
|
28
28
|
export type DenoiseMode = 'article' | 'document';
|
|
29
29
|
/** One denoise pipeline outcome. */
|
|
@@ -31,23 +31,49 @@ export interface DenoiseResult {
|
|
|
31
31
|
/** The markdown body (title, when found, already prepended). */
|
|
32
32
|
markdown: string;
|
|
33
33
|
/**
|
|
34
|
-
* `article` =
|
|
35
|
-
* `document` = whole-document
|
|
36
|
-
*
|
|
34
|
+
* `article` = a content-focused candidate (Readability, main container,
|
|
35
|
+
* record region, forum post stream); `document` = a whole-document
|
|
36
|
+
* candidate (sanitized body, optionally chrome-stripped).
|
|
37
37
|
*/
|
|
38
38
|
mode: DenoiseMode;
|
|
39
|
+
/** The candidate strategy the selector picked. */
|
|
40
|
+
strategy: CandidateKind;
|
|
39
41
|
}
|
|
42
|
+
/** One candidate after sanitization, with its selector inputs. */
|
|
43
|
+
export interface ScoredCandidate {
|
|
44
|
+
kind: CandidateKind;
|
|
45
|
+
title?: string;
|
|
46
|
+
legacy: boolean;
|
|
47
|
+
root: HTMLElement;
|
|
48
|
+
stats: CandidateStats;
|
|
49
|
+
vector: number[];
|
|
50
|
+
score: number;
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Parse, generate, sanitize, featurize, and score every candidate.
|
|
54
|
+
* @returns candidates sorted by the selector's estimate, best first.
|
|
55
|
+
*/
|
|
56
|
+
export declare function scoreCandidates(html: string, url: string): ScoredCandidate[];
|
|
57
|
+
/**
|
|
58
|
+
* Block-classifier training rows for one page (benchmark tooling): every
|
|
59
|
+
* text block's feature vector and its text, for labeling against ground
|
|
60
|
+
* truth.
|
|
61
|
+
*/
|
|
62
|
+
export declare function blockTrainingRows(html: string, url: string): {
|
|
63
|
+
vector: number[];
|
|
64
|
+
text: string;
|
|
65
|
+
}[];
|
|
66
|
+
/** Render one sanitized candidate to markdown (title prepended when found). */
|
|
67
|
+
export declare function renderCandidate(candidate: Pick<ScoredCandidate, 'root' | 'title'>): string;
|
|
40
68
|
/**
|
|
41
69
|
* Convert one rendered HTML document to denoised markdown.
|
|
42
70
|
*
|
|
43
|
-
*
|
|
44
|
-
*
|
|
45
|
-
*
|
|
46
|
-
* cleaner than raw turndown, and a degraded page beats an error for a body
|
|
47
|
-
* the browser already rendered.
|
|
71
|
+
* Every extraction heuristic proposes a candidate (see `./candidates.ts`);
|
|
72
|
+
* the learned selector (`./selector.ts`) estimates each candidate's quality
|
|
73
|
+
* from page- and candidate-level features and the best one is rendered.
|
|
48
74
|
*
|
|
49
75
|
* @param html - the rendered page HTML (`page.content()`).
|
|
50
76
|
* @param url - the page URL, used to resolve relative links during parsing.
|
|
51
|
-
* @returns the markdown
|
|
77
|
+
* @returns the markdown, the extraction mode, and the chosen strategy.
|
|
52
78
|
*/
|
|
53
79
|
export declare function htmlToMarkdown(html: string, url: string): DenoiseResult;
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Learned candidate selector: page-level + candidate-level features → an
|
|
3
|
+
* estimated word-F1 per candidate (gradient-boosted regression trees trained
|
|
4
|
+
* offline by `benchmark/train-selector.py`); the pipeline returns the
|
|
5
|
+
* candidate with the highest estimate.
|
|
6
|
+
*
|
|
7
|
+
* Page features double as a page-type signal (generator meta, JSON-LD
|
|
8
|
+
* `@type`, `og:type`, URL shape, DOM statistics), so the trees learn
|
|
9
|
+
* type-conditional choices — "prune on articles, keep the link directory on
|
|
10
|
+
* docs", "unwrap noscript on forums only" — that no single global threshold
|
|
11
|
+
* could express.
|
|
12
|
+
*
|
|
13
|
+
* @module dsh-web-fetch-playwright/selector
|
|
14
|
+
*/
|
|
15
|
+
import { type CandidateKind, type GenerationFacts } from './candidates.ts';
|
|
16
|
+
import { type TreeEnsemble } from './trees.ts';
|
|
17
|
+
/** Text statistics of one sanitized candidate. */
|
|
18
|
+
export interface CandidateStats {
|
|
19
|
+
kind: CandidateKind;
|
|
20
|
+
legacy: boolean;
|
|
21
|
+
words: number;
|
|
22
|
+
linkWords: number;
|
|
23
|
+
paragraphs: number;
|
|
24
|
+
headings: number;
|
|
25
|
+
listItems: number;
|
|
26
|
+
tableCells: number;
|
|
27
|
+
pres: number;
|
|
28
|
+
images: number;
|
|
29
|
+
blocks: number;
|
|
30
|
+
tokens: Set<string>;
|
|
31
|
+
consentHits: number;
|
|
32
|
+
/** Text blocks as (key, words, linked words), for block-level diffs. */
|
|
33
|
+
blockList: {
|
|
34
|
+
key: string;
|
|
35
|
+
words: number;
|
|
36
|
+
linkWords: number;
|
|
37
|
+
}[];
|
|
38
|
+
}
|
|
39
|
+
/** Measure one sanitized candidate body. */
|
|
40
|
+
export declare function measureCandidate(kind: CandidateKind, legacy: boolean, root: Element): CandidateStats;
|
|
41
|
+
/** Page-level feature names, in vector order. */
|
|
42
|
+
export declare const PAGE_FEATURES: readonly ["body_words_log", "noscript_share", "body_link_density", "readability_null", "gate_pass", "prune_changed", "has_main", "has_preloaded", "record_items_log", "record_share", ...string[], "og_article", "og_product", "og_website", "n_article_log", "n_h1", "n_h2_log", "n_pre_log", "n_time_log", "n_form_log", "n_price_log", "n_itemprop_price", "n_product_itemtype", "n_post_class_log", "n_table_log", "n_code_log", "n_card_class_log"];
|
|
43
|
+
/** Candidate-level feature names, in vector order (one-hot kinds appended). */
|
|
44
|
+
export declare const CANDIDATE_FEATURES: readonly ["words_log", "words_vs_doc", "words_vs_max", "words_vs_art_log", "link_density", "paragraphs_log", "headings_log", "li_share", "td_log", "pre_log", "img_log", "words_per_block", "precision_vs_art", "recall_vs_art", "precision_vs_doc", "consent_hits", "legacy", "extra_share", "extra_link_density", "extra_long_log", "extra_words_per_block", "missing_share", ...string[]];
|
|
45
|
+
export declare const FEATURE_NAMES: string[];
|
|
46
|
+
/** Compute the page feature vector. */
|
|
47
|
+
export declare function pageFeatures(document: Document, url: string, facts: GenerationFacts, doc: CandidateStats | undefined, bodyText: number): number[];
|
|
48
|
+
/** Compute one candidate's feature vector (page vector excluded). */
|
|
49
|
+
export declare function candidateFeatures(stats: CandidateStats, all: CandidateStats[]): number[];
|
|
50
|
+
/** A trained gradient-boosted regressor. */
|
|
51
|
+
export interface SelectorModel extends TreeEnsemble {
|
|
52
|
+
version: string;
|
|
53
|
+
features: readonly string[];
|
|
54
|
+
/** Estimate lead another candidate needs over the legacy pick to replace it. */
|
|
55
|
+
margin?: number;
|
|
56
|
+
}
|
|
57
|
+
/** Estimated quality of one candidate; falls back to the legacy choice without a model. */
|
|
58
|
+
export declare function scoreCandidate(vector: number[], model?: SelectorModel): number;
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Flat gradient-boosted tree inference shared by the candidate selector and
|
|
3
|
+
* the block classifier (models exported from sklearn by the benchmark
|
|
4
|
+
* training scripts).
|
|
5
|
+
*
|
|
6
|
+
* @module dsh-web-fetch-playwright/trees
|
|
7
|
+
*/
|
|
8
|
+
/** One tree in flat array form (sklearn layout: leaf when left === -1). */
|
|
9
|
+
export interface FlatTree {
|
|
10
|
+
feature: number[];
|
|
11
|
+
threshold: number[];
|
|
12
|
+
left: number[];
|
|
13
|
+
right: number[];
|
|
14
|
+
value: number[];
|
|
15
|
+
}
|
|
16
|
+
/** Any additive tree ensemble: `init + learningRate * Σ tree(x)`. */
|
|
17
|
+
export interface TreeEnsemble {
|
|
18
|
+
init: number;
|
|
19
|
+
learningRate: number;
|
|
20
|
+
trees: FlatTree[];
|
|
21
|
+
/** Compare in float64 (HistGradientBoosting) instead of sklearn trees' float32. */
|
|
22
|
+
float64?: boolean;
|
|
23
|
+
}
|
|
24
|
+
/** Raw ensemble output for one feature vector. */
|
|
25
|
+
export declare function predictTrees(model: TreeEnsemble, x: number[]): number;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dsh-web-fetch-playwright",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.20",
|
|
4
4
|
"description": "Playwright/CDP web-fetch provider for DeepSeek Harness: renders pages in a real browser, denoises them (Readability + DOMPurify), and returns markdown.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|