dsh-web-fetch-playwright 0.2.18 → 0.2.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,2 @@
1
+ import type { BlockModel } from './blocks.ts';
2
+ export declare const BLOCK_MODEL: BlockModel;
@@ -0,0 +1,67 @@
1
+ /**
2
+ * Block-level content classification — the `blk` candidate.
3
+ *
4
+ * Readability picks ONE container, so pages whose content is spread across
5
+ * many sections (service pages, product pages with spec blocks) lose recall,
6
+ * while the whole-document fallback keeps everything and loses precision.
7
+ * Here every text block (the text a block element owns directly) is scored
8
+ * independently by gradient-boosted trees (trained offline by
9
+ * `benchmark/train-blocks.py` on WCXB + ScrapingHub ground truth), and the
10
+ * kept blocks are re-emitted in document order — recall and precision are
11
+ * traded per block instead of per container.
12
+ *
13
+ * @module dsh-web-fetch-playwright/blocks
14
+ */
15
+ import { type TreeEnsemble } from './trees.ts';
16
+ /**
17
+ * Word tokens: runs of letters/digits, except that each Han / kana character
18
+ * is its own token. Unspaced scripts would otherwise count a whole sentence
19
+ * as one "word", making a long Chinese paragraph weigh the same as a
20
+ * one-line headline link and breaking every density and ratio feature.
21
+ * Pages without CJK text tokenize exactly as with plain `[\p{L}\p{N}_]+`.
22
+ */
23
+ export declare const WORD: RegExp;
24
+ /** One text block. */
25
+ export interface Block {
26
+ element: Element;
27
+ tag: string;
28
+ text: string;
29
+ words: number;
30
+ linkWords: number;
31
+ links: number;
32
+ depth: number;
33
+ negative: boolean;
34
+ positive: boolean;
35
+ inMain: boolean;
36
+ inRecord: boolean;
37
+ }
38
+ /** Normalized key used to match a block against another extraction's blocks. */
39
+ export declare function blockKey(text: string): string;
40
+ /**
41
+ * Split `root` into text blocks: each text node belongs to its nearest
42
+ * block-level ancestor, and a block is an element with directly owned text.
43
+ */
44
+ export declare function segmentBlocks(root: Element, main?: Element | null, recordItems?: Set<Element>): Block[];
45
+ /** Block-level feature names (page features are prepended). */
46
+ export declare const BLOCK_FEATURES: readonly ["b_words_log", "b_chars_log", "b_link_density", "b_links_log", "b_p", "b_li", "b_h1", "b_hx", "b_td", "b_pre", "b_quote", "b_dd", "b_div", "b_depth", "b_pos", "b_in_art", "b_in_main", "b_in_record", "b_negative", "b_positive", "b_punct", "b_digits", "b_price", "b_prev_in_art", "b_next_in_art", "b_window_words_log", "b_window_link_density", "b_words_vs_max"];
47
+ /** Feature vectors for every block (page vector excluded). */
48
+ export declare function blockFeatures(blocks: Block[], artKeys: Set<string>): number[][];
49
+ /** A trained block classifier (log-odds GBDT). */
50
+ export interface BlockModel extends TreeEnsemble {
51
+ version: string;
52
+ threshold: number;
53
+ }
54
+ /** Override the block model (benchmark cross-fitting only). */
55
+ export declare function setBlockModel(model: BlockModel | null): void;
56
+ /** Whether a block model is available. */
57
+ export declare function hasBlockModel(): boolean;
58
+ /** Probability that each block is content. */
59
+ export declare function classifyBlocks(vectors: number[][]): number[];
60
+ /** Probability cut for keeping a block. */
61
+ export declare function blockThreshold(): number;
62
+ /**
63
+ * Re-emit kept blocks as HTML in document order: headings and `pre` keep
64
+ * their tag, list items are grouped into lists, everything else becomes a
65
+ * paragraph of the block's own inline content.
66
+ */
67
+ export declare function renderBlocks(blocks: Block[], keep: boolean[]): string;
@@ -0,0 +1,157 @@
1
+ /**
2
+ * Candidate generation for the denoise pipeline.
3
+ *
4
+ * The pipeline used to be ONE chain of hard decisions (Readability → coverage
5
+ * gate → trailing prune → whole-document fallback), each tuned by a global
6
+ * threshold. Every page shape that disagrees with a threshold's notion of
7
+ * "content" (link-dense docs indexes vs. recommendation rails, noscript
8
+ * server fallbacks on forums vs. lazy-image wrappers elsewhere) turned each
9
+ * fix into a regression somewhere else. Here every heuristic instead
10
+ * *proposes* a candidate extraction, and a learned selector
11
+ * (`./selector.ts`) picks one per page from page-level and candidate-level
12
+ * features — adding a heuristic can only add an option, never silently
13
+ * override the others.
14
+ *
15
+ * Candidates (see {@link CandidateKind}):
16
+ * - `art` Readability, trailing link farms pruned (the 0.2.18 article path)
17
+ * - `artu` Readability unpruned (link directories that ARE content)
18
+ * - `artx` Readability with a lenient link-density modifier
19
+ * - `main` the page's semantic main container (`<main>`, `[role=main]`, …)
20
+ * - `doc` the whole sanitized document (the 0.2.18 fallback)
21
+ * - `docc` the whole document minus consent/cookie banners, hidden and
22
+ * chrome-classed subtrees
23
+ * - `rec` the dominant repeated-record region (product cards, list rows)
24
+ * - `ns_art` / `ns_doc` the same two paths after unwrapping `<noscript>`
25
+ * server fallbacks (Discourse and friends ship their whole thread
26
+ * there for script-less clients)
27
+ * - `pre` Discourse `data-preloaded` post stream
28
+ * - `blk` blocks kept by the block classifier (`./blocks.ts`), added by
29
+ * the orchestrator once page features exist
30
+ *
31
+ * @module dsh-web-fetch-playwright/candidates
32
+ */
33
+ /** Every candidate extraction strategy, in the selector's one-hot order. */
34
+ export declare const CANDIDATE_KINDS: readonly ["art", "artu", "artx", "main", "doc", "docc", "rec", "ns_art", "ns_doc", "pre", "blk"];
35
+ export type CandidateKind = typeof CANDIDATE_KINDS[number];
36
+ /** One proposed extraction, still as (unsanitized) HTML. */
37
+ export interface RawCandidate {
38
+ kind: CandidateKind;
39
+ /** HTML of the extraction; sanitized by the caller. */
40
+ html: string;
41
+ /** Title to prepend when the body does not open with a heading. */
42
+ title?: string;
43
+ /** Whether the 0.2.18 pipeline would have returned exactly this candidate. */
44
+ legacy: boolean;
45
+ }
46
+ /** Layout/noise tags DOMPurify removes outright (with their content). */
47
+ export declare const FORBID_TAGS: string[];
48
+ /** Tags whose subtrees never count as visible text. */
49
+ export declare const NOISE_SUBTREE_TAGS: Set<string>;
50
+ /**
51
+ * Count non-whitespace characters under `root`, skipping noise subtrees.
52
+ * Whitespace is ignored so CJK and latin pages measure comparable densities.
53
+ */
54
+ export declare function visibleTextLength(root: Node | null, skip?: Set<string>): number;
55
+ /** Count non-whitespace characters under `root` that sit inside an anchor. */
56
+ export declare function linkTextLength(root: Node | null): number;
57
+ /** Minimum characters a trusted Readability extraction carries. */
58
+ export declare const GATE_MIN_ARTICLE_TEXT_CHARS = 250;
59
+ /**
60
+ * The 0.2.18 acceptance gate: distrust an extraction that is tiny in absolute
61
+ * terms or covers a sliver of the page's visible text. Kept as a selector
62
+ * feature and as the legacy-choice marker; it no longer decides alone.
63
+ */
64
+ export declare function articleCoversPage(articleHtml: string, document: Document): boolean;
65
+ /** Prune with the 0.2.18 misfire guard; returns the original on a misfire. */
66
+ export declare function prunedArticleContent(articleHtml: string, document: Document): string;
67
+ /**
68
+ * A copy of `document` whose recommendation rails ("post_recommends",
69
+ * "related-posts", Outbrain/Taboola widgets) carry a marker attribute, or
70
+ * null when it has none. Readability scores class/id names and text, never
71
+ * `data-*`, so it picks the same container on the copy as on the original —
72
+ * removing the rails beforehand instead was measured to shift its pick to a
73
+ * larger ancestor (WCXB 4206).
74
+ */
75
+ export declare function markRecommendationRails(document: Document): Document | null;
76
+ /**
77
+ * Split a Readability extraction of a rail-marked page into the extraction
78
+ * exactly as it would be unmarked, and — when a marked rail follows prose —
79
+ * the extraction without those rails.
80
+ *
81
+ * The trailing prune cannot catch a rail that outweighs the article (on a
82
+ * short 163.com post the rail is 86% of Readability's pick, past the share
83
+ * guards that keep docs indexes intact), and Readability strips the class
84
+ * names that would tell the two apart, hence the marker. A rail is removed
85
+ * only when it is recognised by name AND link density AND sits after some
86
+ * prose inside the extraction: a category page whose "related-product" grid
87
+ * opens the content (WCXB 4247) is the content, not a rail.
88
+ */
89
+ export declare function splitRails(articleHtml: string, document: Document): {
90
+ plain: string;
91
+ derailed: string | null;
92
+ };
93
+ /**
94
+ * An element's element children as an array. Walks sibling pointers:
95
+ * jsdom's `HTMLCollection` is a proxy whose indexed access dominates
96
+ * profiles when `Array.from(el.children)` runs on every element of a large
97
+ * page (measured: 5.2s for a 4,000-paragraph article).
98
+ */
99
+ export declare function childElements(element: Element): Element[];
100
+ /**
101
+ * Rewrite every `<noscript>` into a `<div>` holding its markup.
102
+ *
103
+ * Script-less parsing (jsdom, static snapshots) exposes noscript content as
104
+ * markup that DOMPurify and Readability then delete wholesale — on Discourse
105
+ * that is the entire server-rendered thread, measured 16–36KB per page.
106
+ * Scripting-enabled parsing leaves it as one raw text node, which is
107
+ * reparsed here so both parser modes converge.
108
+ */
109
+ export declare function unwrapNoscripts(document: Document): number;
110
+ /** Visible text inside `<noscript>` elements (parsed either way). */
111
+ export declare function noscriptTextLength(document: Document): number;
112
+ /**
113
+ * Strip page chrome from a cloned body: chrome-named/roled subtrees, hidden
114
+ * subtrees, and consent banners recognised by their wording. A subtree is
115
+ * only removed while it is a minority of the page, so a site that names its
116
+ * article wrapper `content-modal` cannot lose everything.
117
+ */
118
+ export declare function stripPageChrome(body: Element): void;
119
+ /** Record-region discovery result. */
120
+ export interface RecordRegion {
121
+ items: Element[];
122
+ /** Total visible text of the items. */
123
+ text: number;
124
+ }
125
+ /**
126
+ * Find the dominant repeated-record region: the sibling group with the same
127
+ * tag + class signature, at least four members, carrying the most text —
128
+ * product cards on a collection page, rows on a listing. Menu-like groups
129
+ * (tiny items, flat subtrees) are skipped.
130
+ */
131
+ export declare function findRecordRegion(body: Element): RecordRegion | null;
132
+ /**
133
+ * Discourse ships the topic's post stream as JSON in
134
+ * `#data-preloaded[data-preloaded]` (a map of JSON strings). Rebuild the
135
+ * thread as HTML: author, date, cooked post body.
136
+ */
137
+ export declare function discoursePreloadedHtml(document: Document): string | null;
138
+ /** Everything candidate generation learned about the page, for features. */
139
+ export interface GenerationFacts {
140
+ readabilityNull: boolean;
141
+ gatePass: boolean;
142
+ pruneChanged: boolean;
143
+ noscriptText: number;
144
+ record: RecordRegion | null;
145
+ hasMain: boolean;
146
+ hasPreloaded: boolean;
147
+ /** The semantic main container, when found. */
148
+ main: Element | null;
149
+ }
150
+ /**
151
+ * Propose every candidate extraction for one parsed page.
152
+ * @param document - the parsed page (data URIs already elided); not mutated.
153
+ */
154
+ export declare function generateCandidates(document: Document): {
155
+ candidates: RawCandidate[];
156
+ facts: GenerationFacts;
157
+ };
@@ -24,8 +24,12 @@ export type { ChallengeVerdict } from './challenge.ts';
24
24
  export { deepCaptureWalk } from './autoscroll.ts';
25
25
  export type { DeepCapture, DeepCaptureStats } from './autoscroll.ts';
26
26
  export { PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider, WEB_FETCH_CHALLENGE_CODE } from './provider.ts';
27
- export { htmlToMarkdown } from './markdown.ts';
28
- export type { DenoiseMode, DenoiseResult } from './markdown.ts';
27
+ export { blockTrainingRows, htmlToMarkdown, renderCandidate, scoreCandidates } from './markdown.ts';
28
+ export { BLOCK_FEATURES, setBlockModel } from './blocks.ts';
29
+ export { extractPdfMarkdown, isPdfBytes, isPdfContentType, layoutPdfMarkdown, mayBePdf, MAX_PDF_BYTES, MAX_PDF_PAGES, readPdfDocument, WEB_PDF_ENCRYPTED_CODE, WEB_PDF_INVALID_CODE, WEB_PDF_NO_TEXT_CODE, WEB_PDF_TOO_LARGE_CODE } from './pdf.ts';
30
+ export type { PdfDocumentText, PdfExtraction, PdfExtractOptions, PdfOutlineEntry, PdfPageText, PdfTextRun } from './pdf.ts';
31
+ export type { DenoiseMode, DenoiseResult, ScoredCandidate } from './markdown.ts';
32
+ export { FEATURE_NAMES, PAGE_FEATURES } from './selector.ts';
29
33
  /** Cordis plugin name used by loader diagnostics. */
30
34
  export declare const name = "dsh-web-fetch-playwright";
31
35
  /**
@@ -1,21 +1,19 @@
1
1
  /**
2
2
  * The denoise pipeline: rendered HTML → sanitized article HTML → markdown.
3
3
  *
4
- * The classic stack, in order: jsdom parses the page Playwright rendered;
5
- * inline `data:` image payloads are elided to size placeholders (build
6
- * tools inline images as base64, which would otherwise dominate the body);
7
- * Mozilla Readability extracts the article (dropping nav bars, sidebars,
8
- * footers, and ad chrome by scoring link density and text mass) — its result
9
- * must then pass a coverage gate that distrusts tiny or sliver-thin
10
- * extractions, which on list-shaped pages (portals, catalogs) is exactly
11
- * what the link-density penalty leaves behind; an accepted extraction is
12
- * then pruned of trailing link-dense sections — the post-article
13
- * recommendation rails and share bars the scorer's vocabularies cannot name
14
- * (they arrive as bare divs with no class to match); DOMPurify sanitizes
15
- * whatever HTML remains and forbids the layout tags noise lives in;
16
- * Turndown with the GFM plugin converts to markdown using the same style
17
- * options and span-safe table rules as the shipped `dsh-tool-web` renderer,
18
- * so output is consistent with what `web_fetch` produces elsewhere.
4
+ * Since 0.2.20 the pipeline is "propose, then select": jsdom parses the
5
+ * page; inline `data:` image payloads are elided to size placeholders;
6
+ * every extraction heuristic proposes a candidate (`./candidates.ts` —
7
+ * Readability pruned/unpruned/lenient, the semantic main container, the
8
+ * whole document with and without consent/chrome stripping, the repeated
9
+ * record region, noscript-unwrapped variants, the Discourse preloaded post
10
+ * stream) plus the block classifier's pick (`./blocks.ts`); DOMPurify
11
+ * sanitizes each and forbids the layout tags noise lives in; a learned
12
+ * selector (`./selector.ts`) estimates each candidate's quality from page-
13
+ * and candidate-level features and the best one wins, with the 0.2.18
14
+ * choice kept unless another candidate clearly beats it. Turndown with the
15
+ * GFM plugin converts the winner using the same style options and span-safe
16
+ * table rules as the shipped `dsh-tool-web` renderer.
19
17
  *
20
18
  * Synchronous for its callers and unit-tested against fixture pages: jsdom
21
19
  * itself is loaded lazily on the first call — and, inside a dsh profile, only
@@ -24,6 +22,8 @@
24
22
  *
25
23
  * @module dsh-web-fetch-playwright/markdown
26
24
  */
25
+ import { type CandidateKind } from './candidates.ts';
26
+ import { type CandidateStats } from './selector.ts';
27
27
  /** Which extraction path produced a result. */
28
28
  export type DenoiseMode = 'article' | 'document';
29
29
  /** One denoise pipeline outcome. */
@@ -31,23 +31,49 @@ export interface DenoiseResult {
31
31
  /** The markdown body (title, when found, already prepended). */
32
32
  markdown: string;
33
33
  /**
34
- * `article` = Readability extraction that passed the coverage gate;
35
- * `document` = whole-document fallback (Readability returned nothing, or
36
- * the gate distrusted what it returned — list-shaped pages).
34
+ * `article` = a content-focused candidate (Readability, main container,
35
+ * record region, forum post stream); `document` = a whole-document
36
+ * candidate (sanitized body, optionally chrome-stripped).
37
37
  */
38
38
  mode: DenoiseMode;
39
+ /** The candidate strategy the selector picked. */
40
+ strategy: CandidateKind;
39
41
  }
42
+ /** One candidate after sanitization, with its selector inputs. */
43
+ export interface ScoredCandidate {
44
+ kind: CandidateKind;
45
+ title?: string;
46
+ legacy: boolean;
47
+ root: HTMLElement;
48
+ stats: CandidateStats;
49
+ vector: number[];
50
+ score: number;
51
+ }
52
+ /**
53
+ * Parse, generate, sanitize, featurize, and score every candidate.
54
+ * @returns candidates sorted by the selector's estimate, best first.
55
+ */
56
+ export declare function scoreCandidates(html: string, url: string): ScoredCandidate[];
57
+ /**
58
+ * Block-classifier training rows for one page (benchmark tooling): every
59
+ * text block's feature vector and its text, for labeling against ground
60
+ * truth.
61
+ */
62
+ export declare function blockTrainingRows(html: string, url: string): {
63
+ vector: number[];
64
+ text: string;
65
+ }[];
66
+ /** Render one sanitized candidate to markdown (title prepended when found). */
67
+ export declare function renderCandidate(candidate: Pick<ScoredCandidate, 'root' | 'title'>): string;
40
68
  /**
41
69
  * Convert one rendered HTML document to denoised markdown.
42
70
  *
43
- * Readability failure (non-article pages, or extractions the coverage gate
44
- * {@link articleCoversPage} distrusts) degrades to converting the sanitized
45
- * whole document — layout tags are still forbidden, so the fallback stays
46
- * cleaner than raw turndown, and a degraded page beats an error for a body
47
- * the browser already rendered.
71
+ * Every extraction heuristic proposes a candidate (see `./candidates.ts`);
72
+ * the learned selector (`./selector.ts`) estimates each candidate's quality
73
+ * from page- and candidate-level features and the best one is rendered.
48
74
  *
49
75
  * @param html - the rendered page HTML (`page.content()`).
50
76
  * @param url - the page URL, used to resolve relative links during parsing.
51
- * @returns the markdown and the extraction mode used.
77
+ * @returns the markdown, the extraction mode, and the chosen strategy.
52
78
  */
53
79
  export declare function htmlToMarkdown(html: string, url: string): DenoiseResult;
@@ -0,0 +1,143 @@
1
+ /**
2
+ * PDF → markdown. A fetched URL that turns out to be a PDF (served inline as
3
+ * `application/pdf`, as a `Content-Disposition: attachment` download, or as
4
+ * an anonymous `application/octet-stream` whose bytes start with `%PDF-`) is
5
+ * routed here instead of failing with `WEB_UNSUPPORTED_CONTENT_TYPE`.
6
+ *
7
+ * Two layers, kept apart so the heuristics are testable without pdf.js:
8
+ *
9
+ * 1. {@link readPdfDocument} — the pdf.js boundary (through `unpdf`'s
10
+ * serverless build, loaded lazily so plugin start never pays for it).
11
+ * Per page it reads the positioned text runs (`getTextContent`) plus the
12
+ * real font names (`getOperatorList` + `fontExtraProperties`, the only way
13
+ * pdf.js exposes "NimbusRomNo9L-Medi" instead of a generic family), so the
14
+ * layout pass can see bold and monospace. Bounded by the fetch deadline,
15
+ * the caller's abort signal, a page cap, and a collected-character cap.
16
+ * 2. {@link layoutPdfMarkdown} — pure heuristics over those runs:
17
+ * - runs → lines (baseline grouping in content-stream order, x-sorted
18
+ * inside a line; rotated runs such as arXiv's side stamp are dropped);
19
+ * - running headers/footers (text repeated in the page margins across
20
+ * pages, digits normalized) and bare page numbers are removed;
21
+ * - the dominant (body) font size and line pitch are measured;
22
+ * - headings come from, strongest first: the PDF outline (bookmarks),
23
+ * section numbering ("3.1 Pre-training", "第二章", "一、"), larger font
24
+ * sizes (ranked into levels), and standalone bold lines;
25
+ * - lines are merged into paragraphs (vertical gap, first-line indent,
26
+ * short terminal line, font change; continuation across columns and
27
+ * pages when the sentence is unfinished), with dictionary-checked
28
+ * de-hyphenation and CJK-aware joining;
29
+ * - bullets / numbered items become markdown lists, monospace blocks
30
+ * become fenced code (unless the whole document is monospace), lines
31
+ * with wide column gaps become GFM tables when they line up;
32
+ * - `<!-- page N -->` markers keep page provenance for citations.
33
+ *
34
+ * Scanned PDFs without a text layer, encrypted PDFs, and broken files fail
35
+ * with dedicated, provider-specific error codes (OCR is out of scope).
36
+ *
37
+ * @module dsh-web-fetch-playwright/pdf
38
+ */
39
+ /** Largest PDF this provider downloads and parses (bytes). */
40
+ export declare const MAX_PDF_BYTES: number;
41
+ /** Most pages read from one document; later pages are reported as truncated. */
42
+ export declare const MAX_PDF_PAGES = 500;
43
+ /** No text layer (a scanned / image-only PDF); OCR is not supported. */
44
+ export declare const WEB_PDF_NO_TEXT_CODE = "WEB_PDF_NO_TEXT";
45
+ /** Password-protected PDF the parser cannot open. */
46
+ export declare const WEB_PDF_ENCRYPTED_CODE = "WEB_PDF_ENCRYPTED";
47
+ /** Bytes claimed (or sniffed) to be a PDF that pdf.js cannot parse. */
48
+ export declare const WEB_PDF_INVALID_CODE = "WEB_PDF_INVALID";
49
+ /** PDF larger than {@link MAX_PDF_BYTES}. */
50
+ export declare const WEB_PDF_TOO_LARGE_CODE = "WEB_PDF_TOO_LARGE";
51
+ /** Whether a `Content-Type` names a PDF. */
52
+ export declare function isPdfContentType(contentType: string | undefined): boolean;
53
+ /**
54
+ * Whether a response that is not otherwise decodable is worth downloading
55
+ * to sniff for PDF bytes: a PDF content type, a `.pdf` attachment filename,
56
+ * a `.pdf` URL path, or an opaque binary type (servers commonly hand PDFs out
57
+ * as `application/octet-stream`). The magic-byte check then decides.
58
+ */
59
+ export declare function mayBePdf(headers: Record<string, string>, url: string): boolean;
60
+ /**
61
+ * Whether the bytes are a PDF: `%PDF-` within the first 1024 bytes (the
62
+ * spec-tolerated leading-garbage window every mainstream reader accepts).
63
+ */
64
+ export declare function isPdfBytes(bytes: Uint8Array): boolean;
65
+ /** One positioned, horizontal text run (pdf.js text item, normalized). */
66
+ export interface PdfTextRun {
67
+ text: string;
68
+ /** Left edge in PDF user space. */
69
+ x: number;
70
+ /** Baseline in PDF user space (origin bottom-left). */
71
+ y: number;
72
+ width: number;
73
+ /** Font size (vertical scale of the text matrix). */
74
+ size: number;
75
+ bold: boolean;
76
+ mono: boolean;
77
+ }
78
+ /** One page's text runs in content-stream order. */
79
+ export interface PdfPageText {
80
+ /** 1-based page number. */
81
+ number: number;
82
+ /** Page box bottom edge (user space). */
83
+ bottom: number;
84
+ /** Page box height (user space). */
85
+ height: number;
86
+ runs: PdfTextRun[];
87
+ }
88
+ /** One bookmark, flattened with its nesting depth (0 = top level). */
89
+ export interface PdfOutlineEntry {
90
+ title: string;
91
+ depth: number;
92
+ }
93
+ /** Everything the layout pass consumes. */
94
+ export interface PdfDocumentText {
95
+ /** Document info / XMP title, when present. */
96
+ title?: string;
97
+ outline: PdfOutlineEntry[];
98
+ pages: PdfPageText[];
99
+ totalPages: number;
100
+ }
101
+ /** The finished extraction. */
102
+ export interface PdfExtraction {
103
+ markdown: string;
104
+ totalPages: number;
105
+ /** Pages actually read (fewer than `totalPages` when a limit cut in). */
106
+ pagesRead: number;
107
+ /** True when a page, size, or time limit stopped the extraction early. */
108
+ truncated: boolean;
109
+ }
110
+ /** Limits one extraction runs under. */
111
+ export interface PdfExtractOptions {
112
+ /** Abort → stop between pages and fail. */
113
+ signal?: AbortSignal;
114
+ /** Wall-clock budget for reading pages (ms); pages left over are truncated. */
115
+ timeBudgetMs?: number;
116
+ /** Stop reading pages once this many characters were collected. */
117
+ maxChars?: number;
118
+ /** Page cap (defaults to {@link MAX_PDF_PAGES}). */
119
+ maxPages?: number;
120
+ }
121
+ /**
122
+ * Read a PDF's positioned text with pdf.js, page by page, inside the given
123
+ * limits. Text items become {@link PdfTextRun}s (horizontal only — rotated
124
+ * stamps and vertical margin text are dropped as layout noise).
125
+ *
126
+ * @returns the text model, plus whether a limit stopped reading early.
127
+ */
128
+ export declare function readPdfDocument(bytes: Uint8Array, options?: PdfExtractOptions): Promise<{
129
+ doc: PdfDocumentText;
130
+ truncated: boolean;
131
+ }>;
132
+ /**
133
+ * Extract a PDF as markdown: read it with pdf.js inside the limits, lay it
134
+ * out, and fail with {@link WEB_PDF_NO_TEXT_CODE} when there is no text layer
135
+ * to speak of (a scan). Sizes beyond {@link MAX_PDF_BYTES} are refused.
136
+ */
137
+ export declare function extractPdfMarkdown(bytes: Uint8Array, options?: PdfExtractOptions): Promise<PdfExtraction>;
138
+ /**
139
+ * Lay a PDF's positioned text out as markdown (see the module docs for the
140
+ * heuristics). Pure — no pdf.js — so the rules are unit-testable with
141
+ * synthetic runs.
142
+ */
143
+ export declare function layoutPdfMarkdown(doc: PdfDocumentText): string;
@@ -33,6 +33,15 @@
33
33
  * {@link DEFAULT_MAX_CONCURRENCY_CDP} tabs); further fetches wait briefly in
34
34
  * a queue and fail fast (rather than hang until abort) when no slot frees.
35
35
  *
36
+ * PDFs: a URL that serves a PDF — committed inline into Chrome's viewer,
37
+ * handed to the download manager as an attachment, or served as an opaque
38
+ * `application/octet-stream` whose bytes start with `%PDF-` — is read as
39
+ * bytes (the navigation body, else a refetch through the page's API request
40
+ * context) and extracted to markdown by `src/pdf.ts` (pdf.js via `unpdf`
41
+ * plus layout heuristics: headings, paragraphs, lists, tables, page
42
+ * markers). Scans without a text layer, encrypted, oversized, and broken
43
+ * PDFs fail with the `WEB_PDF_*` codes.
44
+ *
36
45
  * Private-network and SSRF protection is not implemented (same stance as the
37
46
  * shipped HTTP provider); a page this provider can reach is whatever the
38
47
  * browser can reach. Profile mode additionally acts WITH the remote
@@ -0,0 +1,2 @@
1
+ import type { SelectorModel } from './selector.ts';
2
+ export declare const SELECTOR_MODEL: SelectorModel;
@@ -0,0 +1,58 @@
1
+ /**
2
+ * Learned candidate selector: page-level + candidate-level features → an
3
+ * estimated word-F1 per candidate (gradient-boosted regression trees trained
4
+ * offline by `benchmark/train-selector.py`); the pipeline returns the
5
+ * candidate with the highest estimate.
6
+ *
7
+ * Page features double as a page-type signal (generator meta, JSON-LD
8
+ * `@type`, `og:type`, URL shape, DOM statistics), so the trees learn
9
+ * type-conditional choices — "prune on articles, keep the link directory on
10
+ * docs", "unwrap noscript on forums only" — that no single global threshold
11
+ * could express.
12
+ *
13
+ * @module dsh-web-fetch-playwright/selector
14
+ */
15
+ import { type CandidateKind, type GenerationFacts } from './candidates.ts';
16
+ import { type TreeEnsemble } from './trees.ts';
17
+ /** Text statistics of one sanitized candidate. */
18
+ export interface CandidateStats {
19
+ kind: CandidateKind;
20
+ legacy: boolean;
21
+ words: number;
22
+ linkWords: number;
23
+ paragraphs: number;
24
+ headings: number;
25
+ listItems: number;
26
+ tableCells: number;
27
+ pres: number;
28
+ images: number;
29
+ blocks: number;
30
+ tokens: Set<string>;
31
+ consentHits: number;
32
+ /** Text blocks as (key, words, linked words), for block-level diffs. */
33
+ blockList: {
34
+ key: string;
35
+ words: number;
36
+ linkWords: number;
37
+ }[];
38
+ }
39
+ /** Measure one sanitized candidate body. */
40
+ export declare function measureCandidate(kind: CandidateKind, legacy: boolean, root: Element): CandidateStats;
41
+ /** Page-level feature names, in vector order. */
42
+ export declare const PAGE_FEATURES: readonly ["body_words_log", "noscript_share", "body_link_density", "readability_null", "gate_pass", "prune_changed", "has_main", "has_preloaded", "record_items_log", "record_share", ...string[], "og_article", "og_product", "og_website", "n_article_log", "n_h1", "n_h2_log", "n_pre_log", "n_time_log", "n_form_log", "n_price_log", "n_itemprop_price", "n_product_itemtype", "n_post_class_log", "n_table_log", "n_code_log", "n_card_class_log"];
43
+ /** Candidate-level feature names, in vector order (one-hot kinds appended). */
44
+ export declare const CANDIDATE_FEATURES: readonly ["words_log", "words_vs_doc", "words_vs_max", "words_vs_art_log", "link_density", "paragraphs_log", "headings_log", "li_share", "td_log", "pre_log", "img_log", "words_per_block", "precision_vs_art", "recall_vs_art", "precision_vs_doc", "consent_hits", "legacy", "extra_share", "extra_link_density", "extra_long_log", "extra_words_per_block", "missing_share", ...string[]];
45
+ export declare const FEATURE_NAMES: string[];
46
+ /** Compute the page feature vector. */
47
+ export declare function pageFeatures(document: Document, url: string, facts: GenerationFacts, doc: CandidateStats | undefined, bodyText: number): number[];
48
+ /** Compute one candidate's feature vector (page vector excluded). */
49
+ export declare function candidateFeatures(stats: CandidateStats, all: CandidateStats[]): number[];
50
+ /** A trained gradient-boosted regressor. */
51
+ export interface SelectorModel extends TreeEnsemble {
52
+ version: string;
53
+ features: readonly string[];
54
+ /** Estimate lead another candidate needs over the legacy pick to replace it. */
55
+ margin?: number;
56
+ }
57
+ /** Estimated quality of one candidate; falls back to the legacy choice without a model. */
58
+ export declare function scoreCandidate(vector: number[], model?: SelectorModel): number;
@@ -0,0 +1,25 @@
1
+ /**
2
+ * Flat gradient-boosted tree inference shared by the candidate selector and
3
+ * the block classifier (models exported from sklearn by the benchmark
4
+ * training scripts).
5
+ *
6
+ * @module dsh-web-fetch-playwright/trees
7
+ */
8
+ /** One tree in flat array form (sklearn layout: leaf when left === -1). */
9
+ export interface FlatTree {
10
+ feature: number[];
11
+ threshold: number[];
12
+ left: number[];
13
+ right: number[];
14
+ value: number[];
15
+ }
16
+ /** Any additive tree ensemble: `init + learningRate * Σ tree(x)`. */
17
+ export interface TreeEnsemble {
18
+ init: number;
19
+ learningRate: number;
20
+ trees: FlatTree[];
21
+ /** Compare in float64 (HistGradientBoosting) instead of sklearn trees' float32. */
22
+ float64?: boolean;
23
+ }
24
+ /** Raw ensemble output for one feature vector. */
25
+ export declare function predictTrees(model: TreeEnsemble, x: number[]): number;
@@ -12,6 +12,11 @@ export interface PlaywrightResponse {
12
12
  status(): number;
13
13
  headers(): Record<string, string>;
14
14
  text(): Promise<string>;
15
+ /**
16
+ * The raw body bytes — the PDF path reads binary bodies through it (a
17
+ * Node `Buffer` is a `Uint8Array`). Absent on minimal fakes.
18
+ */
19
+ body?(): Promise<Uint8Array>;
15
20
  /** This response's URL; absent on minimal fakes. */
16
21
  url?(): string;
17
22
  /**
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "dsh-web-fetch-playwright",
3
- "version": "0.2.18",
3
+ "version": "0.2.20",
4
4
  "description": "Playwright/CDP web-fetch provider for DeepSeek Harness: renders pages in a real browser, denoises them (Readability + DOMPurify), and returns markdown.",
5
5
  "type": "module",
6
6
  "license": "MIT",
@@ -93,7 +93,8 @@
93
93
  "dompurify": "^3.2.4",
94
94
  "jsdom": "^26.0.0",
95
95
  "playwright-core": "^1.65.0",
96
- "turndown": "^7.2.0"
96
+ "turndown": "^7.2.0",
97
+ "unpdf": "^2.1.0"
97
98
  },
98
99
  "peerDependencies": {
99
100
  "@deepseek-ai/cordis": ">=4.0.1-rc.1 <5",