dsh-web-fetch-playwright 0.2.18 → 0.2.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -1
- package/README.zh-CN.md +17 -1
- package/lib/client.js +18 -18
- package/lib/index.js +58203 -423
- package/lib/types/block-model.d.ts +2 -0
- package/lib/types/blocks.d.ts +67 -0
- package/lib/types/candidates.d.ts +157 -0
- package/lib/types/index.d.ts +6 -2
- package/lib/types/markdown.d.ts +50 -24
- package/lib/types/pdf.d.ts +143 -0
- package/lib/types/provider.d.ts +9 -0
- package/lib/types/selector-model.d.ts +2 -0
- package/lib/types/selector.d.ts +58 -0
- package/lib/types/trees.d.ts +25 -0
- package/lib/types/types.d.ts +5 -0
- package/package.json +3 -2
- package/src/block-model.ts +5 -0
- package/src/blocks.ts +223 -0
- package/src/candidates.ts +639 -0
- package/src/index.ts +6 -2
- package/src/markdown.ts +136 -413
- package/src/pdf.ts +1114 -0
- package/src/provider.ts +117 -8
- package/src/selector-model.ts +6 -0
- package/src/selector.ts +277 -0
- package/src/session-window.ts +8 -0
- package/src/trees.ts +43 -0
- package/src/types.ts +5 -0
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Block-level content classification — the `blk` candidate.
|
|
3
|
+
*
|
|
4
|
+
* Readability picks ONE container, so pages whose content is spread across
|
|
5
|
+
* many sections (service pages, product pages with spec blocks) lose recall,
|
|
6
|
+
* while the whole-document fallback keeps everything and loses precision.
|
|
7
|
+
* Here every text block (the text a block element owns directly) is scored
|
|
8
|
+
* independently by gradient-boosted trees (trained offline by
|
|
9
|
+
* `benchmark/train-blocks.py` on WCXB + ScrapingHub ground truth), and the
|
|
10
|
+
* kept blocks are re-emitted in document order — recall and precision are
|
|
11
|
+
* traded per block instead of per container.
|
|
12
|
+
*
|
|
13
|
+
* @module dsh-web-fetch-playwright/blocks
|
|
14
|
+
*/
|
|
15
|
+
import { type TreeEnsemble } from './trees.ts';
|
|
16
|
+
/**
|
|
17
|
+
* Word tokens: runs of letters/digits, except that each Han / kana character
|
|
18
|
+
* is its own token. Unspaced scripts would otherwise count a whole sentence
|
|
19
|
+
* as one "word", making a long Chinese paragraph weigh the same as a
|
|
20
|
+
* one-line headline link and breaking every density and ratio feature.
|
|
21
|
+
* Pages without CJK text tokenize exactly as with plain `[\p{L}\p{N}_]+`.
|
|
22
|
+
*/
|
|
23
|
+
export declare const WORD: RegExp;
|
|
24
|
+
/** One text block. */
|
|
25
|
+
export interface Block {
|
|
26
|
+
element: Element;
|
|
27
|
+
tag: string;
|
|
28
|
+
text: string;
|
|
29
|
+
words: number;
|
|
30
|
+
linkWords: number;
|
|
31
|
+
links: number;
|
|
32
|
+
depth: number;
|
|
33
|
+
negative: boolean;
|
|
34
|
+
positive: boolean;
|
|
35
|
+
inMain: boolean;
|
|
36
|
+
inRecord: boolean;
|
|
37
|
+
}
|
|
38
|
+
/** Normalized key used to match a block against another extraction's blocks. */
|
|
39
|
+
export declare function blockKey(text: string): string;
|
|
40
|
+
/**
|
|
41
|
+
* Split `root` into text blocks: each text node belongs to its nearest
|
|
42
|
+
* block-level ancestor, and a block is an element with directly owned text.
|
|
43
|
+
*/
|
|
44
|
+
export declare function segmentBlocks(root: Element, main?: Element | null, recordItems?: Set<Element>): Block[];
|
|
45
|
+
/** Block-level feature names (page features are prepended). */
|
|
46
|
+
export declare const BLOCK_FEATURES: readonly ["b_words_log", "b_chars_log", "b_link_density", "b_links_log", "b_p", "b_li", "b_h1", "b_hx", "b_td", "b_pre", "b_quote", "b_dd", "b_div", "b_depth", "b_pos", "b_in_art", "b_in_main", "b_in_record", "b_negative", "b_positive", "b_punct", "b_digits", "b_price", "b_prev_in_art", "b_next_in_art", "b_window_words_log", "b_window_link_density", "b_words_vs_max"];
|
|
47
|
+
/** Feature vectors for every block (page vector excluded). */
|
|
48
|
+
export declare function blockFeatures(blocks: Block[], artKeys: Set<string>): number[][];
|
|
49
|
+
/** A trained block classifier (log-odds GBDT). */
|
|
50
|
+
export interface BlockModel extends TreeEnsemble {
|
|
51
|
+
version: string;
|
|
52
|
+
threshold: number;
|
|
53
|
+
}
|
|
54
|
+
/** Override the block model (benchmark cross-fitting only). */
|
|
55
|
+
export declare function setBlockModel(model: BlockModel | null): void;
|
|
56
|
+
/** Whether a block model is available. */
|
|
57
|
+
export declare function hasBlockModel(): boolean;
|
|
58
|
+
/** Probability that each block is content. */
|
|
59
|
+
export declare function classifyBlocks(vectors: number[][]): number[];
|
|
60
|
+
/** Probability cut for keeping a block. */
|
|
61
|
+
export declare function blockThreshold(): number;
|
|
62
|
+
/**
|
|
63
|
+
* Re-emit kept blocks as HTML in document order: headings and `pre` keep
|
|
64
|
+
* their tag, list items are grouped into lists, everything else becomes a
|
|
65
|
+
* paragraph of the block's own inline content.
|
|
66
|
+
*/
|
|
67
|
+
export declare function renderBlocks(blocks: Block[], keep: boolean[]): string;
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Candidate generation for the denoise pipeline.
|
|
3
|
+
*
|
|
4
|
+
* The pipeline used to be ONE chain of hard decisions (Readability → coverage
|
|
5
|
+
* gate → trailing prune → whole-document fallback), each tuned by a global
|
|
6
|
+
* threshold. Every page shape that disagrees with a threshold's notion of
|
|
7
|
+
* "content" (link-dense docs indexes vs. recommendation rails, noscript
|
|
8
|
+
* server fallbacks on forums vs. lazy-image wrappers elsewhere) turned each
|
|
9
|
+
* fix into a regression somewhere else. Here every heuristic instead
|
|
10
|
+
* *proposes* a candidate extraction, and a learned selector
|
|
11
|
+
* (`./selector.ts`) picks one per page from page-level and candidate-level
|
|
12
|
+
* features — adding a heuristic can only add an option, never silently
|
|
13
|
+
* override the others.
|
|
14
|
+
*
|
|
15
|
+
* Candidates (see {@link CandidateKind}):
|
|
16
|
+
* - `art` Readability, trailing link farms pruned (the 0.2.18 article path)
|
|
17
|
+
* - `artu` Readability unpruned (link directories that ARE content)
|
|
18
|
+
* - `artx` Readability with a lenient link-density modifier
|
|
19
|
+
* - `main` the page's semantic main container (`<main>`, `[role=main]`, …)
|
|
20
|
+
* - `doc` the whole sanitized document (the 0.2.18 fallback)
|
|
21
|
+
* - `docc` the whole document minus consent/cookie banners, hidden and
|
|
22
|
+
* chrome-classed subtrees
|
|
23
|
+
* - `rec` the dominant repeated-record region (product cards, list rows)
|
|
24
|
+
* - `ns_art` / `ns_doc` the same two paths after unwrapping `<noscript>`
|
|
25
|
+
* server fallbacks (Discourse and friends ship their whole thread
|
|
26
|
+
* there for script-less clients)
|
|
27
|
+
* - `pre` Discourse `data-preloaded` post stream
|
|
28
|
+
* - `blk` blocks kept by the block classifier (`./blocks.ts`), added by
|
|
29
|
+
* the orchestrator once page features exist
|
|
30
|
+
*
|
|
31
|
+
* @module dsh-web-fetch-playwright/candidates
|
|
32
|
+
*/
|
|
33
|
+
/** Every candidate extraction strategy, in the selector's one-hot order. */
|
|
34
|
+
export declare const CANDIDATE_KINDS: readonly ["art", "artu", "artx", "main", "doc", "docc", "rec", "ns_art", "ns_doc", "pre", "blk"];
|
|
35
|
+
export type CandidateKind = typeof CANDIDATE_KINDS[number];
|
|
36
|
+
/** One proposed extraction, still as (unsanitized) HTML. */
|
|
37
|
+
export interface RawCandidate {
|
|
38
|
+
kind: CandidateKind;
|
|
39
|
+
/** HTML of the extraction; sanitized by the caller. */
|
|
40
|
+
html: string;
|
|
41
|
+
/** Title to prepend when the body does not open with a heading. */
|
|
42
|
+
title?: string;
|
|
43
|
+
/** Whether the 0.2.18 pipeline would have returned exactly this candidate. */
|
|
44
|
+
legacy: boolean;
|
|
45
|
+
}
|
|
46
|
+
/** Layout/noise tags DOMPurify removes outright (with their content). */
|
|
47
|
+
export declare const FORBID_TAGS: string[];
|
|
48
|
+
/** Tags whose subtrees never count as visible text. */
|
|
49
|
+
export declare const NOISE_SUBTREE_TAGS: Set<string>;
|
|
50
|
+
/**
|
|
51
|
+
* Count non-whitespace characters under `root`, skipping noise subtrees.
|
|
52
|
+
* Whitespace is ignored so CJK and latin pages measure comparable densities.
|
|
53
|
+
*/
|
|
54
|
+
export declare function visibleTextLength(root: Node | null, skip?: Set<string>): number;
|
|
55
|
+
/** Count non-whitespace characters under `root` that sit inside an anchor. */
|
|
56
|
+
export declare function linkTextLength(root: Node | null): number;
|
|
57
|
+
/** Minimum characters a trusted Readability extraction carries. */
|
|
58
|
+
export declare const GATE_MIN_ARTICLE_TEXT_CHARS = 250;
|
|
59
|
+
/**
|
|
60
|
+
* The 0.2.18 acceptance gate: distrust an extraction that is tiny in absolute
|
|
61
|
+
* terms or covers a sliver of the page's visible text. Kept as a selector
|
|
62
|
+
* feature and as the legacy-choice marker; it no longer decides alone.
|
|
63
|
+
*/
|
|
64
|
+
export declare function articleCoversPage(articleHtml: string, document: Document): boolean;
|
|
65
|
+
/** Prune with the 0.2.18 misfire guard; returns the original on a misfire. */
|
|
66
|
+
export declare function prunedArticleContent(articleHtml: string, document: Document): string;
|
|
67
|
+
/**
|
|
68
|
+
* A copy of `document` whose recommendation rails ("post_recommends",
|
|
69
|
+
* "related-posts", Outbrain/Taboola widgets) carry a marker attribute, or
|
|
70
|
+
* null when it has none. Readability scores class/id names and text, never
|
|
71
|
+
* `data-*`, so it picks the same container on the copy as on the original —
|
|
72
|
+
* removing the rails beforehand instead was measured to shift its pick to a
|
|
73
|
+
* larger ancestor (WCXB 4206).
|
|
74
|
+
*/
|
|
75
|
+
export declare function markRecommendationRails(document: Document): Document | null;
|
|
76
|
+
/**
|
|
77
|
+
* Split a Readability extraction of a rail-marked page into the extraction
|
|
78
|
+
* exactly as it would be unmarked, and — when a marked rail follows prose —
|
|
79
|
+
* the extraction without those rails.
|
|
80
|
+
*
|
|
81
|
+
* The trailing prune cannot catch a rail that outweighs the article (on a
|
|
82
|
+
* short 163.com post the rail is 86% of Readability's pick, past the share
|
|
83
|
+
* guards that keep docs indexes intact), and Readability strips the class
|
|
84
|
+
* names that would tell the two apart, hence the marker. A rail is removed
|
|
85
|
+
* only when it is recognised by name AND link density AND sits after some
|
|
86
|
+
* prose inside the extraction: a category page whose "related-product" grid
|
|
87
|
+
* opens the content (WCXB 4247) is the content, not a rail.
|
|
88
|
+
*/
|
|
89
|
+
export declare function splitRails(articleHtml: string, document: Document): {
|
|
90
|
+
plain: string;
|
|
91
|
+
derailed: string | null;
|
|
92
|
+
};
|
|
93
|
+
/**
|
|
94
|
+
* An element's element children as an array. Walks sibling pointers:
|
|
95
|
+
* jsdom's `HTMLCollection` is a proxy whose indexed access dominates
|
|
96
|
+
* profiles when `Array.from(el.children)` runs on every element of a large
|
|
97
|
+
* page (measured: 5.2s for a 4,000-paragraph article).
|
|
98
|
+
*/
|
|
99
|
+
export declare function childElements(element: Element): Element[];
|
|
100
|
+
/**
|
|
101
|
+
* Rewrite every `<noscript>` into a `<div>` holding its markup.
|
|
102
|
+
*
|
|
103
|
+
* Script-less parsing (jsdom, static snapshots) exposes noscript content as
|
|
104
|
+
* markup that DOMPurify and Readability then delete wholesale — on Discourse
|
|
105
|
+
* that is the entire server-rendered thread, measured 16–36KB per page.
|
|
106
|
+
* Scripting-enabled parsing leaves it as one raw text node, which is
|
|
107
|
+
* reparsed here so both parser modes converge.
|
|
108
|
+
*/
|
|
109
|
+
export declare function unwrapNoscripts(document: Document): number;
|
|
110
|
+
/** Visible text inside `<noscript>` elements (parsed either way). */
|
|
111
|
+
export declare function noscriptTextLength(document: Document): number;
|
|
112
|
+
/**
|
|
113
|
+
* Strip page chrome from a cloned body: chrome-named/roled subtrees, hidden
|
|
114
|
+
* subtrees, and consent banners recognised by their wording. A subtree is
|
|
115
|
+
* only removed while it is a minority of the page, so a site that names its
|
|
116
|
+
* article wrapper `content-modal` cannot lose everything.
|
|
117
|
+
*/
|
|
118
|
+
export declare function stripPageChrome(body: Element): void;
|
|
119
|
+
/** Record-region discovery result. */
|
|
120
|
+
export interface RecordRegion {
|
|
121
|
+
items: Element[];
|
|
122
|
+
/** Total visible text of the items. */
|
|
123
|
+
text: number;
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* Find the dominant repeated-record region: the sibling group with the same
|
|
127
|
+
* tag + class signature, at least four members, carrying the most text —
|
|
128
|
+
* product cards on a collection page, rows on a listing. Menu-like groups
|
|
129
|
+
* (tiny items, flat subtrees) are skipped.
|
|
130
|
+
*/
|
|
131
|
+
export declare function findRecordRegion(body: Element): RecordRegion | null;
|
|
132
|
+
/**
|
|
133
|
+
* Discourse ships the topic's post stream as JSON in
|
|
134
|
+
* `#data-preloaded[data-preloaded]` (a map of JSON strings). Rebuild the
|
|
135
|
+
* thread as HTML: author, date, cooked post body.
|
|
136
|
+
*/
|
|
137
|
+
export declare function discoursePreloadedHtml(document: Document): string | null;
|
|
138
|
+
/** Everything candidate generation learned about the page, for features. */
|
|
139
|
+
export interface GenerationFacts {
|
|
140
|
+
readabilityNull: boolean;
|
|
141
|
+
gatePass: boolean;
|
|
142
|
+
pruneChanged: boolean;
|
|
143
|
+
noscriptText: number;
|
|
144
|
+
record: RecordRegion | null;
|
|
145
|
+
hasMain: boolean;
|
|
146
|
+
hasPreloaded: boolean;
|
|
147
|
+
/** The semantic main container, when found. */
|
|
148
|
+
main: Element | null;
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* Propose every candidate extraction for one parsed page.
|
|
152
|
+
* @param document - the parsed page (data URIs already elided); not mutated.
|
|
153
|
+
*/
|
|
154
|
+
export declare function generateCandidates(document: Document): {
|
|
155
|
+
candidates: RawCandidate[];
|
|
156
|
+
facts: GenerationFacts;
|
|
157
|
+
};
|
package/lib/types/index.d.ts
CHANGED
|
@@ -24,8 +24,12 @@ export type { ChallengeVerdict } from './challenge.ts';
|
|
|
24
24
|
export { deepCaptureWalk } from './autoscroll.ts';
|
|
25
25
|
export type { DeepCapture, DeepCaptureStats } from './autoscroll.ts';
|
|
26
26
|
export { PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider, WEB_FETCH_CHALLENGE_CODE } from './provider.ts';
|
|
27
|
-
export { htmlToMarkdown } from './markdown.ts';
|
|
28
|
-
export
|
|
27
|
+
export { blockTrainingRows, htmlToMarkdown, renderCandidate, scoreCandidates } from './markdown.ts';
|
|
28
|
+
export { BLOCK_FEATURES, setBlockModel } from './blocks.ts';
|
|
29
|
+
export { extractPdfMarkdown, isPdfBytes, isPdfContentType, layoutPdfMarkdown, mayBePdf, MAX_PDF_BYTES, MAX_PDF_PAGES, readPdfDocument, WEB_PDF_ENCRYPTED_CODE, WEB_PDF_INVALID_CODE, WEB_PDF_NO_TEXT_CODE, WEB_PDF_TOO_LARGE_CODE } from './pdf.ts';
|
|
30
|
+
export type { PdfDocumentText, PdfExtraction, PdfExtractOptions, PdfOutlineEntry, PdfPageText, PdfTextRun } from './pdf.ts';
|
|
31
|
+
export type { DenoiseMode, DenoiseResult, ScoredCandidate } from './markdown.ts';
|
|
32
|
+
export { FEATURE_NAMES, PAGE_FEATURES } from './selector.ts';
|
|
29
33
|
/** Cordis plugin name used by loader diagnostics. */
|
|
30
34
|
export declare const name = "dsh-web-fetch-playwright";
|
|
31
35
|
/**
|
package/lib/types/markdown.d.ts
CHANGED
|
@@ -1,21 +1,19 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* The denoise pipeline: rendered HTML → sanitized article HTML → markdown.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
* inline `data:` image payloads are elided to size placeholders
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
* options and span-safe table rules as the shipped `dsh-tool-web` renderer,
|
|
18
|
-
* so output is consistent with what `web_fetch` produces elsewhere.
|
|
4
|
+
* Since 0.2.20 the pipeline is "propose, then select": jsdom parses the
|
|
5
|
+
* page; inline `data:` image payloads are elided to size placeholders;
|
|
6
|
+
* every extraction heuristic proposes a candidate (`./candidates.ts` —
|
|
7
|
+
* Readability pruned/unpruned/lenient, the semantic main container, the
|
|
8
|
+
* whole document with and without consent/chrome stripping, the repeated
|
|
9
|
+
* record region, noscript-unwrapped variants, the Discourse preloaded post
|
|
10
|
+
* stream) plus the block classifier's pick (`./blocks.ts`); DOMPurify
|
|
11
|
+
* sanitizes each and forbids the layout tags noise lives in; a learned
|
|
12
|
+
* selector (`./selector.ts`) estimates each candidate's quality from page-
|
|
13
|
+
* and candidate-level features and the best one wins, with the 0.2.18
|
|
14
|
+
* choice kept unless another candidate clearly beats it. Turndown with the
|
|
15
|
+
* GFM plugin converts the winner using the same style options and span-safe
|
|
16
|
+
* table rules as the shipped `dsh-tool-web` renderer.
|
|
19
17
|
*
|
|
20
18
|
* Synchronous for its callers and unit-tested against fixture pages: jsdom
|
|
21
19
|
* itself is loaded lazily on the first call — and, inside a dsh profile, only
|
|
@@ -24,6 +22,8 @@
|
|
|
24
22
|
*
|
|
25
23
|
* @module dsh-web-fetch-playwright/markdown
|
|
26
24
|
*/
|
|
25
|
+
import { type CandidateKind } from './candidates.ts';
|
|
26
|
+
import { type CandidateStats } from './selector.ts';
|
|
27
27
|
/** Which extraction path produced a result. */
|
|
28
28
|
export type DenoiseMode = 'article' | 'document';
|
|
29
29
|
/** One denoise pipeline outcome. */
|
|
@@ -31,23 +31,49 @@ export interface DenoiseResult {
|
|
|
31
31
|
/** The markdown body (title, when found, already prepended). */
|
|
32
32
|
markdown: string;
|
|
33
33
|
/**
|
|
34
|
-
* `article` =
|
|
35
|
-
* `document` = whole-document
|
|
36
|
-
*
|
|
34
|
+
* `article` = a content-focused candidate (Readability, main container,
|
|
35
|
+
* record region, forum post stream); `document` = a whole-document
|
|
36
|
+
* candidate (sanitized body, optionally chrome-stripped).
|
|
37
37
|
*/
|
|
38
38
|
mode: DenoiseMode;
|
|
39
|
+
/** The candidate strategy the selector picked. */
|
|
40
|
+
strategy: CandidateKind;
|
|
39
41
|
}
|
|
42
|
+
/** One candidate after sanitization, with its selector inputs. */
|
|
43
|
+
export interface ScoredCandidate {
|
|
44
|
+
kind: CandidateKind;
|
|
45
|
+
title?: string;
|
|
46
|
+
legacy: boolean;
|
|
47
|
+
root: HTMLElement;
|
|
48
|
+
stats: CandidateStats;
|
|
49
|
+
vector: number[];
|
|
50
|
+
score: number;
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Parse, generate, sanitize, featurize, and score every candidate.
|
|
54
|
+
* @returns candidates sorted by the selector's estimate, best first.
|
|
55
|
+
*/
|
|
56
|
+
export declare function scoreCandidates(html: string, url: string): ScoredCandidate[];
|
|
57
|
+
/**
|
|
58
|
+
* Block-classifier training rows for one page (benchmark tooling): every
|
|
59
|
+
* text block's feature vector and its text, for labeling against ground
|
|
60
|
+
* truth.
|
|
61
|
+
*/
|
|
62
|
+
export declare function blockTrainingRows(html: string, url: string): {
|
|
63
|
+
vector: number[];
|
|
64
|
+
text: string;
|
|
65
|
+
}[];
|
|
66
|
+
/** Render one sanitized candidate to markdown (title prepended when found). */
|
|
67
|
+
export declare function renderCandidate(candidate: Pick<ScoredCandidate, 'root' | 'title'>): string;
|
|
40
68
|
/**
|
|
41
69
|
* Convert one rendered HTML document to denoised markdown.
|
|
42
70
|
*
|
|
43
|
-
*
|
|
44
|
-
*
|
|
45
|
-
*
|
|
46
|
-
* cleaner than raw turndown, and a degraded page beats an error for a body
|
|
47
|
-
* the browser already rendered.
|
|
71
|
+
* Every extraction heuristic proposes a candidate (see `./candidates.ts`);
|
|
72
|
+
* the learned selector (`./selector.ts`) estimates each candidate's quality
|
|
73
|
+
* from page- and candidate-level features and the best one is rendered.
|
|
48
74
|
*
|
|
49
75
|
* @param html - the rendered page HTML (`page.content()`).
|
|
50
76
|
* @param url - the page URL, used to resolve relative links during parsing.
|
|
51
|
-
* @returns the markdown
|
|
77
|
+
* @returns the markdown, the extraction mode, and the chosen strategy.
|
|
52
78
|
*/
|
|
53
79
|
export declare function htmlToMarkdown(html: string, url: string): DenoiseResult;
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* PDF → markdown. A fetched URL that turns out to be a PDF (served inline as
|
|
3
|
+
* `application/pdf`, as a `Content-Disposition: attachment` download, or as
|
|
4
|
+
* an anonymous `application/octet-stream` whose bytes start with `%PDF-`) is
|
|
5
|
+
* routed here instead of failing with `WEB_UNSUPPORTED_CONTENT_TYPE`.
|
|
6
|
+
*
|
|
7
|
+
* Two layers, kept apart so the heuristics are testable without pdf.js:
|
|
8
|
+
*
|
|
9
|
+
* 1. {@link readPdfDocument} — the pdf.js boundary (through `unpdf`'s
|
|
10
|
+
* serverless build, loaded lazily so plugin start never pays for it).
|
|
11
|
+
* Per page it reads the positioned text runs (`getTextContent`) plus the
|
|
12
|
+
* real font names (`getOperatorList` + `fontExtraProperties`, the only way
|
|
13
|
+
* pdf.js exposes "NimbusRomNo9L-Medi" instead of a generic family), so the
|
|
14
|
+
* layout pass can see bold and monospace. Bounded by the fetch deadline,
|
|
15
|
+
* the caller's abort signal, a page cap, and a collected-character cap.
|
|
16
|
+
* 2. {@link layoutPdfMarkdown} — pure heuristics over those runs:
|
|
17
|
+
* - runs → lines (baseline grouping in content-stream order, x-sorted
|
|
18
|
+
* inside a line; rotated runs such as arXiv's side stamp are dropped);
|
|
19
|
+
* - running headers/footers (text repeated in the page margins across
|
|
20
|
+
* pages, digits normalized) and bare page numbers are removed;
|
|
21
|
+
* - the dominant (body) font size and line pitch are measured;
|
|
22
|
+
* - headings come from, strongest first: the PDF outline (bookmarks),
|
|
23
|
+
* section numbering ("3.1 Pre-training", "第二章", "一、"), larger font
|
|
24
|
+
* sizes (ranked into levels), and standalone bold lines;
|
|
25
|
+
* - lines are merged into paragraphs (vertical gap, first-line indent,
|
|
26
|
+
* short terminal line, font change; continuation across columns and
|
|
27
|
+
* pages when the sentence is unfinished), with dictionary-checked
|
|
28
|
+
* de-hyphenation and CJK-aware joining;
|
|
29
|
+
* - bullets / numbered items become markdown lists, monospace blocks
|
|
30
|
+
* become fenced code (unless the whole document is monospace), lines
|
|
31
|
+
* with wide column gaps become GFM tables when they line up;
|
|
32
|
+
* - `<!-- page N -->` markers keep page provenance for citations.
|
|
33
|
+
*
|
|
34
|
+
* Scanned PDFs without a text layer, encrypted PDFs, and broken files fail
|
|
35
|
+
* with dedicated, provider-specific error codes (OCR is out of scope).
|
|
36
|
+
*
|
|
37
|
+
* @module dsh-web-fetch-playwright/pdf
|
|
38
|
+
*/
|
|
39
|
+
/** Largest PDF this provider downloads and parses (bytes). */
|
|
40
|
+
export declare const MAX_PDF_BYTES: number;
|
|
41
|
+
/** Most pages read from one document; later pages are reported as truncated. */
|
|
42
|
+
export declare const MAX_PDF_PAGES = 500;
|
|
43
|
+
/** No text layer (a scanned / image-only PDF); OCR is not supported. */
|
|
44
|
+
export declare const WEB_PDF_NO_TEXT_CODE = "WEB_PDF_NO_TEXT";
|
|
45
|
+
/** Password-protected PDF the parser cannot open. */
|
|
46
|
+
export declare const WEB_PDF_ENCRYPTED_CODE = "WEB_PDF_ENCRYPTED";
|
|
47
|
+
/** Bytes claimed (or sniffed) to be a PDF that pdf.js cannot parse. */
|
|
48
|
+
export declare const WEB_PDF_INVALID_CODE = "WEB_PDF_INVALID";
|
|
49
|
+
/** PDF larger than {@link MAX_PDF_BYTES}. */
|
|
50
|
+
export declare const WEB_PDF_TOO_LARGE_CODE = "WEB_PDF_TOO_LARGE";
|
|
51
|
+
/** Whether a `Content-Type` names a PDF. */
|
|
52
|
+
export declare function isPdfContentType(contentType: string | undefined): boolean;
|
|
53
|
+
/**
|
|
54
|
+
* Whether a response that is not otherwise decodable is worth downloading
|
|
55
|
+
* to sniff for PDF bytes: a PDF content type, a `.pdf` attachment filename,
|
|
56
|
+
* a `.pdf` URL path, or an opaque binary type (servers commonly hand PDFs out
|
|
57
|
+
* as `application/octet-stream`). The magic-byte check then decides.
|
|
58
|
+
*/
|
|
59
|
+
export declare function mayBePdf(headers: Record<string, string>, url: string): boolean;
|
|
60
|
+
/**
|
|
61
|
+
* Whether the bytes are a PDF: `%PDF-` within the first 1024 bytes (the
|
|
62
|
+
* spec-tolerated leading-garbage window every mainstream reader accepts).
|
|
63
|
+
*/
|
|
64
|
+
export declare function isPdfBytes(bytes: Uint8Array): boolean;
|
|
65
|
+
/** One positioned, horizontal text run (pdf.js text item, normalized). */
|
|
66
|
+
export interface PdfTextRun {
|
|
67
|
+
text: string;
|
|
68
|
+
/** Left edge in PDF user space. */
|
|
69
|
+
x: number;
|
|
70
|
+
/** Baseline in PDF user space (origin bottom-left). */
|
|
71
|
+
y: number;
|
|
72
|
+
width: number;
|
|
73
|
+
/** Font size (vertical scale of the text matrix). */
|
|
74
|
+
size: number;
|
|
75
|
+
bold: boolean;
|
|
76
|
+
mono: boolean;
|
|
77
|
+
}
|
|
78
|
+
/** One page's text runs in content-stream order. */
|
|
79
|
+
export interface PdfPageText {
|
|
80
|
+
/** 1-based page number. */
|
|
81
|
+
number: number;
|
|
82
|
+
/** Page box bottom edge (user space). */
|
|
83
|
+
bottom: number;
|
|
84
|
+
/** Page box height (user space). */
|
|
85
|
+
height: number;
|
|
86
|
+
runs: PdfTextRun[];
|
|
87
|
+
}
|
|
88
|
+
/** One bookmark, flattened with its nesting depth (0 = top level). */
|
|
89
|
+
export interface PdfOutlineEntry {
|
|
90
|
+
title: string;
|
|
91
|
+
depth: number;
|
|
92
|
+
}
|
|
93
|
+
/** Everything the layout pass consumes. */
|
|
94
|
+
export interface PdfDocumentText {
|
|
95
|
+
/** Document info / XMP title, when present. */
|
|
96
|
+
title?: string;
|
|
97
|
+
outline: PdfOutlineEntry[];
|
|
98
|
+
pages: PdfPageText[];
|
|
99
|
+
totalPages: number;
|
|
100
|
+
}
|
|
101
|
+
/** The finished extraction. */
|
|
102
|
+
export interface PdfExtraction {
|
|
103
|
+
markdown: string;
|
|
104
|
+
totalPages: number;
|
|
105
|
+
/** Pages actually read (fewer than `totalPages` when a limit cut in). */
|
|
106
|
+
pagesRead: number;
|
|
107
|
+
/** True when a page, size, or time limit stopped the extraction early. */
|
|
108
|
+
truncated: boolean;
|
|
109
|
+
}
|
|
110
|
+
/** Limits one extraction runs under. */
|
|
111
|
+
export interface PdfExtractOptions {
|
|
112
|
+
/** Abort → stop between pages and fail. */
|
|
113
|
+
signal?: AbortSignal;
|
|
114
|
+
/** Wall-clock budget for reading pages (ms); pages left over are truncated. */
|
|
115
|
+
timeBudgetMs?: number;
|
|
116
|
+
/** Stop reading pages once this many characters were collected. */
|
|
117
|
+
maxChars?: number;
|
|
118
|
+
/** Page cap (defaults to {@link MAX_PDF_PAGES}). */
|
|
119
|
+
maxPages?: number;
|
|
120
|
+
}
|
|
121
|
+
/**
|
|
122
|
+
* Read a PDF's positioned text with pdf.js, page by page, inside the given
|
|
123
|
+
* limits. Text items become {@link PdfTextRun}s (horizontal only — rotated
|
|
124
|
+
* stamps and vertical margin text are dropped as layout noise).
|
|
125
|
+
*
|
|
126
|
+
* @returns the text model, plus whether a limit stopped reading early.
|
|
127
|
+
*/
|
|
128
|
+
export declare function readPdfDocument(bytes: Uint8Array, options?: PdfExtractOptions): Promise<{
|
|
129
|
+
doc: PdfDocumentText;
|
|
130
|
+
truncated: boolean;
|
|
131
|
+
}>;
|
|
132
|
+
/**
|
|
133
|
+
* Extract a PDF as markdown: read it with pdf.js inside the limits, lay it
|
|
134
|
+
* out, and fail with {@link WEB_PDF_NO_TEXT_CODE} when there is no text layer
|
|
135
|
+
* to speak of (a scan). Sizes beyond {@link MAX_PDF_BYTES} are refused.
|
|
136
|
+
*/
|
|
137
|
+
export declare function extractPdfMarkdown(bytes: Uint8Array, options?: PdfExtractOptions): Promise<PdfExtraction>;
|
|
138
|
+
/**
|
|
139
|
+
* Lay a PDF's positioned text out as markdown (see the module docs for the
|
|
140
|
+
* heuristics). Pure — no pdf.js — so the rules are unit-testable with
|
|
141
|
+
* synthetic runs.
|
|
142
|
+
*/
|
|
143
|
+
export declare function layoutPdfMarkdown(doc: PdfDocumentText): string;
|
package/lib/types/provider.d.ts
CHANGED
|
@@ -33,6 +33,15 @@
|
|
|
33
33
|
* {@link DEFAULT_MAX_CONCURRENCY_CDP} tabs); further fetches wait briefly in
|
|
34
34
|
* a queue and fail fast (rather than hang until abort) when no slot frees.
|
|
35
35
|
*
|
|
36
|
+
* PDFs: a URL that serves a PDF — committed inline into Chrome's viewer,
|
|
37
|
+
* handed to the download manager as an attachment, or served as an opaque
|
|
38
|
+
* `application/octet-stream` whose bytes start with `%PDF-` — is read as
|
|
39
|
+
* bytes (the navigation body, else a refetch through the page's API request
|
|
40
|
+
* context) and extracted to markdown by `src/pdf.ts` (pdf.js via `unpdf`
|
|
41
|
+
* plus layout heuristics: headings, paragraphs, lists, tables, page
|
|
42
|
+
* markers). Scans without a text layer, encrypted, oversized, and broken
|
|
43
|
+
* PDFs fail with the `WEB_PDF_*` codes.
|
|
44
|
+
*
|
|
36
45
|
* Private-network and SSRF protection is not implemented (same stance as the
|
|
37
46
|
* shipped HTTP provider); a page this provider can reach is whatever the
|
|
38
47
|
* browser can reach. Profile mode additionally acts WITH the remote
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Learned candidate selector: page-level + candidate-level features → an
|
|
3
|
+
* estimated word-F1 per candidate (gradient-boosted regression trees trained
|
|
4
|
+
* offline by `benchmark/train-selector.py`); the pipeline returns the
|
|
5
|
+
* candidate with the highest estimate.
|
|
6
|
+
*
|
|
7
|
+
* Page features double as a page-type signal (generator meta, JSON-LD
|
|
8
|
+
* `@type`, `og:type`, URL shape, DOM statistics), so the trees learn
|
|
9
|
+
* type-conditional choices — "prune on articles, keep the link directory on
|
|
10
|
+
* docs", "unwrap noscript on forums only" — that no single global threshold
|
|
11
|
+
* could express.
|
|
12
|
+
*
|
|
13
|
+
* @module dsh-web-fetch-playwright/selector
|
|
14
|
+
*/
|
|
15
|
+
import { type CandidateKind, type GenerationFacts } from './candidates.ts';
|
|
16
|
+
import { type TreeEnsemble } from './trees.ts';
|
|
17
|
+
/** Text statistics of one sanitized candidate. */
|
|
18
|
+
export interface CandidateStats {
|
|
19
|
+
kind: CandidateKind;
|
|
20
|
+
legacy: boolean;
|
|
21
|
+
words: number;
|
|
22
|
+
linkWords: number;
|
|
23
|
+
paragraphs: number;
|
|
24
|
+
headings: number;
|
|
25
|
+
listItems: number;
|
|
26
|
+
tableCells: number;
|
|
27
|
+
pres: number;
|
|
28
|
+
images: number;
|
|
29
|
+
blocks: number;
|
|
30
|
+
tokens: Set<string>;
|
|
31
|
+
consentHits: number;
|
|
32
|
+
/** Text blocks as (key, words, linked words), for block-level diffs. */
|
|
33
|
+
blockList: {
|
|
34
|
+
key: string;
|
|
35
|
+
words: number;
|
|
36
|
+
linkWords: number;
|
|
37
|
+
}[];
|
|
38
|
+
}
|
|
39
|
+
/** Measure one sanitized candidate body. */
|
|
40
|
+
export declare function measureCandidate(kind: CandidateKind, legacy: boolean, root: Element): CandidateStats;
|
|
41
|
+
/** Page-level feature names, in vector order. */
|
|
42
|
+
export declare const PAGE_FEATURES: readonly ["body_words_log", "noscript_share", "body_link_density", "readability_null", "gate_pass", "prune_changed", "has_main", "has_preloaded", "record_items_log", "record_share", ...string[], "og_article", "og_product", "og_website", "n_article_log", "n_h1", "n_h2_log", "n_pre_log", "n_time_log", "n_form_log", "n_price_log", "n_itemprop_price", "n_product_itemtype", "n_post_class_log", "n_table_log", "n_code_log", "n_card_class_log"];
|
|
43
|
+
/** Candidate-level feature names, in vector order (one-hot kinds appended). */
|
|
44
|
+
export declare const CANDIDATE_FEATURES: readonly ["words_log", "words_vs_doc", "words_vs_max", "words_vs_art_log", "link_density", "paragraphs_log", "headings_log", "li_share", "td_log", "pre_log", "img_log", "words_per_block", "precision_vs_art", "recall_vs_art", "precision_vs_doc", "consent_hits", "legacy", "extra_share", "extra_link_density", "extra_long_log", "extra_words_per_block", "missing_share", ...string[]];
|
|
45
|
+
export declare const FEATURE_NAMES: string[];
|
|
46
|
+
/** Compute the page feature vector. */
|
|
47
|
+
export declare function pageFeatures(document: Document, url: string, facts: GenerationFacts, doc: CandidateStats | undefined, bodyText: number): number[];
|
|
48
|
+
/** Compute one candidate's feature vector (page vector excluded). */
|
|
49
|
+
export declare function candidateFeatures(stats: CandidateStats, all: CandidateStats[]): number[];
|
|
50
|
+
/** A trained gradient-boosted regressor. */
|
|
51
|
+
export interface SelectorModel extends TreeEnsemble {
|
|
52
|
+
version: string;
|
|
53
|
+
features: readonly string[];
|
|
54
|
+
/** Estimate lead another candidate needs over the legacy pick to replace it. */
|
|
55
|
+
margin?: number;
|
|
56
|
+
}
|
|
57
|
+
/** Estimated quality of one candidate; falls back to the legacy choice without a model. */
|
|
58
|
+
export declare function scoreCandidate(vector: number[], model?: SelectorModel): number;
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Flat gradient-boosted tree inference shared by the candidate selector and
|
|
3
|
+
* the block classifier (models exported from sklearn by the benchmark
|
|
4
|
+
* training scripts).
|
|
5
|
+
*
|
|
6
|
+
* @module dsh-web-fetch-playwright/trees
|
|
7
|
+
*/
|
|
8
|
+
/** One tree in flat array form (sklearn layout: leaf when left === -1). */
|
|
9
|
+
export interface FlatTree {
|
|
10
|
+
feature: number[];
|
|
11
|
+
threshold: number[];
|
|
12
|
+
left: number[];
|
|
13
|
+
right: number[];
|
|
14
|
+
value: number[];
|
|
15
|
+
}
|
|
16
|
+
/** Any additive tree ensemble: `init + learningRate * Σ tree(x)`. */
|
|
17
|
+
export interface TreeEnsemble {
|
|
18
|
+
init: number;
|
|
19
|
+
learningRate: number;
|
|
20
|
+
trees: FlatTree[];
|
|
21
|
+
/** Compare in float64 (HistGradientBoosting) instead of sklearn trees' float32. */
|
|
22
|
+
float64?: boolean;
|
|
23
|
+
}
|
|
24
|
+
/** Raw ensemble output for one feature vector. */
|
|
25
|
+
export declare function predictTrees(model: TreeEnsemble, x: number[]): number;
|
package/lib/types/types.d.ts
CHANGED
|
@@ -12,6 +12,11 @@ export interface PlaywrightResponse {
|
|
|
12
12
|
status(): number;
|
|
13
13
|
headers(): Record<string, string>;
|
|
14
14
|
text(): Promise<string>;
|
|
15
|
+
/**
|
|
16
|
+
* The raw body bytes — the PDF path reads binary bodies through it (a
|
|
17
|
+
* Node `Buffer` is a `Uint8Array`). Absent on minimal fakes.
|
|
18
|
+
*/
|
|
19
|
+
body?(): Promise<Uint8Array>;
|
|
15
20
|
/** This response's URL; absent on minimal fakes. */
|
|
16
21
|
url?(): string;
|
|
17
22
|
/**
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dsh-web-fetch-playwright",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.20",
|
|
4
4
|
"description": "Playwright/CDP web-fetch provider for DeepSeek Harness: renders pages in a real browser, denoises them (Readability + DOMPurify), and returns markdown.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -93,7 +93,8 @@
|
|
|
93
93
|
"dompurify": "^3.2.4",
|
|
94
94
|
"jsdom": "^26.0.0",
|
|
95
95
|
"playwright-core": "^1.65.0",
|
|
96
|
-
"turndown": "^7.2.0"
|
|
96
|
+
"turndown": "^7.2.0",
|
|
97
|
+
"unpdf": "^2.1.0"
|
|
97
98
|
},
|
|
98
99
|
"peerDependencies": {
|
|
99
100
|
"@deepseek-ai/cordis": ">=4.0.1-rc.1 <5",
|