dsh-web-fetch-playwright 0.2.19 → 0.2.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -15
- package/README.zh-CN.md +15 -15
- package/cordis.patch.yml +0 -4
- package/lib/client.js +23 -73
- package/lib/index.js +58188 -448
- package/lib/types/autoscroll.d.ts +46 -10
- package/lib/types/block-model.d.ts +2 -0
- package/lib/types/blocks.d.ts +67 -0
- package/lib/types/candidates.d.ts +164 -0
- package/lib/types/client/controller.d.ts +0 -14
- package/lib/types/client/locales.d.ts +1 -1
- package/lib/types/config.d.ts +54 -30
- package/lib/types/index.d.ts +5 -3
- package/lib/types/markdown.d.ts +50 -24
- package/lib/types/provider.d.ts +46 -9
- package/lib/types/selector-model.d.ts +2 -0
- package/lib/types/selector.d.ts +58 -0
- package/lib/types/trees.d.ts +25 -0
- package/lib/types/types.d.ts +7 -6
- package/package.json +1 -1
- package/src/autoscroll.ts +459 -17
- package/src/block-model.ts +5 -0
- package/src/blocks.ts +223 -0
- package/src/candidates.ts +655 -0
- package/src/client/card.tsx +0 -26
- package/src/client/controller.ts +0 -18
- package/src/client/locales.ts +0 -14
- package/src/config.ts +67 -38
- package/src/index.ts +6 -3
- package/src/markdown.ts +136 -413
- package/src/provider.ts +198 -37
- package/src/selector-model.ts +6 -0
- package/src/selector.ts +288 -0
- package/src/trees.ts +43 -0
- package/src/types.ts +7 -7
package/lib/types/markdown.d.ts
CHANGED
|
@@ -1,21 +1,19 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* The denoise pipeline: rendered HTML → sanitized article HTML → markdown.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
* inline `data:` image payloads are elided to size placeholders
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
* options and span-safe table rules as the shipped `dsh-tool-web` renderer,
|
|
18
|
-
* so output is consistent with what `web_fetch` produces elsewhere.
|
|
4
|
+
* Since 0.2.20 the pipeline is "propose, then select": jsdom parses the
|
|
5
|
+
* page; inline `data:` image payloads are elided to size placeholders;
|
|
6
|
+
* every extraction heuristic proposes a candidate (`./candidates.ts` —
|
|
7
|
+
* Readability pruned/unpruned/lenient, the semantic main container, the
|
|
8
|
+
* whole document with and without consent/chrome stripping, the repeated
|
|
9
|
+
* record region, noscript-unwrapped variants, the Discourse preloaded post
|
|
10
|
+
* stream) plus the block classifier's pick (`./blocks.ts`); DOMPurify
|
|
11
|
+
* sanitizes each and forbids the layout tags noise lives in; a learned
|
|
12
|
+
* selector (`./selector.ts`) estimates each candidate's quality from page-
|
|
13
|
+
* and candidate-level features and the best one wins, with the 0.2.18
|
|
14
|
+
* choice kept unless another candidate clearly beats it. Turndown with the
|
|
15
|
+
* GFM plugin converts the winner using the same style options and span-safe
|
|
16
|
+
* table rules as the shipped `dsh-tool-web` renderer.
|
|
19
17
|
*
|
|
20
18
|
* Synchronous for its callers and unit-tested against fixture pages: jsdom
|
|
21
19
|
* itself is loaded lazily on the first call — and, inside a dsh profile, only
|
|
@@ -24,6 +22,8 @@
|
|
|
24
22
|
*
|
|
25
23
|
* @module dsh-web-fetch-playwright/markdown
|
|
26
24
|
*/
|
|
25
|
+
import { type CandidateKind } from './candidates.ts';
|
|
26
|
+
import { type CandidateStats } from './selector.ts';
|
|
27
27
|
/** Which extraction path produced a result. */
|
|
28
28
|
export type DenoiseMode = 'article' | 'document';
|
|
29
29
|
/** One denoise pipeline outcome. */
|
|
@@ -31,23 +31,49 @@ export interface DenoiseResult {
|
|
|
31
31
|
/** The markdown body (title, when found, already prepended). */
|
|
32
32
|
markdown: string;
|
|
33
33
|
/**
|
|
34
|
-
* `article` =
|
|
35
|
-
* `document` = whole-document
|
|
36
|
-
*
|
|
34
|
+
* `article` = a content-focused candidate (Readability, main container,
|
|
35
|
+
* record region, forum post stream); `document` = a whole-document
|
|
36
|
+
* candidate (sanitized body, optionally chrome-stripped).
|
|
37
37
|
*/
|
|
38
38
|
mode: DenoiseMode;
|
|
39
|
+
/** The candidate strategy the selector picked. */
|
|
40
|
+
strategy: CandidateKind;
|
|
39
41
|
}
|
|
42
|
+
/** One candidate after sanitization, with its selector inputs. */
|
|
43
|
+
export interface ScoredCandidate {
|
|
44
|
+
kind: CandidateKind;
|
|
45
|
+
title?: string;
|
|
46
|
+
legacy: boolean;
|
|
47
|
+
root: HTMLElement;
|
|
48
|
+
stats: CandidateStats;
|
|
49
|
+
vector: number[];
|
|
50
|
+
score: number;
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Parse, generate, sanitize, featurize, and score every candidate.
|
|
54
|
+
* @returns candidates sorted by the selector's estimate, best first.
|
|
55
|
+
*/
|
|
56
|
+
export declare function scoreCandidates(html: string, url: string): ScoredCandidate[];
|
|
57
|
+
/**
|
|
58
|
+
* Block-classifier training rows for one page (benchmark tooling): every
|
|
59
|
+
* text block's feature vector and its text, for labeling against ground
|
|
60
|
+
* truth.
|
|
61
|
+
*/
|
|
62
|
+
export declare function blockTrainingRows(html: string, url: string): {
|
|
63
|
+
vector: number[];
|
|
64
|
+
text: string;
|
|
65
|
+
}[];
|
|
66
|
+
/** Render one sanitized candidate to markdown (title prepended when found). */
|
|
67
|
+
export declare function renderCandidate(candidate: Pick<ScoredCandidate, 'root' | 'title'>): string;
|
|
40
68
|
/**
|
|
41
69
|
* Convert one rendered HTML document to denoised markdown.
|
|
42
70
|
*
|
|
43
|
-
*
|
|
44
|
-
*
|
|
45
|
-
*
|
|
46
|
-
* cleaner than raw turndown, and a degraded page beats an error for a body
|
|
47
|
-
* the browser already rendered.
|
|
71
|
+
* Every extraction heuristic proposes a candidate (see `./candidates.ts`);
|
|
72
|
+
* the learned selector (`./selector.ts`) estimates each candidate's quality
|
|
73
|
+
* from page- and candidate-level features and the best one is rendered.
|
|
48
74
|
*
|
|
49
75
|
* @param html - the rendered page HTML (`page.content()`).
|
|
50
76
|
* @param url - the page URL, used to resolve relative links during parsing.
|
|
51
|
-
* @returns the markdown
|
|
77
|
+
* @returns the markdown, the extraction mode, and the chosen strategy.
|
|
52
78
|
*/
|
|
53
79
|
export declare function htmlToMarkdown(html: string, url: string): DenoiseResult;
|
package/lib/types/provider.d.ts
CHANGED
|
@@ -55,9 +55,14 @@
|
|
|
55
55
|
* behavior). The wait tracks the LAST main-frame navigation response (the
|
|
56
56
|
* real page reloads in after the challenge clears) and watches the live DOM
|
|
57
57
|
* so SPA-style clears are caught too. It never clicks, never injects
|
|
58
|
-
* answers, never touches cookies itself; when the budget runs out
|
|
59
|
-
*
|
|
60
|
-
*
|
|
58
|
+
* answers, never touches cookies itself; when the budget runs out, ONE
|
|
59
|
+
* plain-HTTP refetch of the same URL through the page's request context
|
|
60
|
+
* (same cookie jar, the browser's UA, none of Chromium's render decisions
|
|
61
|
+
* — `challengeHttpFallback`, default on) tries to retrieve the real
|
|
62
|
+
* document, gated by the same response-level and content-level challenge
|
|
63
|
+
* classifiers so an interstitial can never pass as content. When that too
|
|
64
|
+
* fails, the fetch fails with {@link WEB_FETCH_CHALLENGE_CODE} instead of
|
|
65
|
+
* returning the interstitial as content.
|
|
61
66
|
*
|
|
62
67
|
* Deep capture (viewport-virtualized feeds and app-shell documents): feed-
|
|
63
68
|
* shaped pages — Discourse topic streams, comment threads, timelines —
|
|
@@ -84,6 +89,17 @@
|
|
|
84
89
|
* degrades to the legacy capture (see `src/autoscroll.ts` for the measured
|
|
85
90
|
* linux.do and Feishu badcases).
|
|
86
91
|
*
|
|
92
|
+
* Deadline (soft, then hard): each fetch has a {@link DEFAULT_TIMEOUT_MS}
|
|
93
|
+
* budget for WAITING — navigation, settle, challenge wait, scroll walk. When
|
|
94
|
+
* it runs out the page is NOT torn down: every wait is bounded by the same
|
|
95
|
+
* budget and simply ends, a navigation still loading is stopped
|
|
96
|
+
* (`window.stop()`) once its main-frame response has arrived, and whatever
|
|
97
|
+
* the page holds by then goes through the normal extraction — a slow or
|
|
98
|
+
* endless page still hands the agent the content it already rendered
|
|
99
|
+
* (flagged `truncated` when the document never finished loading). Only when
|
|
100
|
+
* nothing arrived at all, or extraction itself hangs past
|
|
101
|
+
* {@link EXTRACT_GRACE_MS}, does the fetch fail with `WEB_FETCH_TIMEOUT`.
|
|
102
|
+
*
|
|
87
103
|
* @module dsh-web-fetch-playwright/provider
|
|
88
104
|
*/
|
|
89
105
|
import type { WebFetchProvider, WebFetchRequest, WebFetchResult } from '@deepseek-ai/dsh-web';
|
|
@@ -119,19 +135,25 @@ export interface BrowserSession {
|
|
|
119
135
|
release?: () => Promise<void>;
|
|
120
136
|
}
|
|
121
137
|
/**
|
|
122
|
-
*
|
|
123
|
-
*
|
|
124
|
-
*
|
|
138
|
+
* The fetch's two-stage deadline composed with the caller's signal. The SOFT
|
|
139
|
+
* stage (`timeoutMs`) only bounds waiting: {@link remainingMs} counts down to
|
|
140
|
+
* it and {@link expired} reports it, so every wait ends there and the fetch
|
|
141
|
+
* extracts what the page holds. The HARD stage (`timeoutMs + graceMs`) aborts
|
|
142
|
+
* {@link signal}, which closes the page. The timeout reason is kept apart
|
|
143
|
+
* from an outer abort so the error translation can pick `WEB_FETCH_TIMEOUT`
|
|
144
|
+
* over `WEB_ABORTED`.
|
|
125
145
|
*/
|
|
126
146
|
declare class Deadline {
|
|
127
147
|
readonly signal: AbortSignal;
|
|
128
148
|
private readonly controller;
|
|
129
149
|
private readonly expiresAt;
|
|
130
150
|
private timedOut;
|
|
131
|
-
constructor(outer: AbortSignal | undefined, timeoutMs: number);
|
|
132
|
-
/** Whether OUR timer fired (vs an outer cancellation). */
|
|
151
|
+
constructor(outer: AbortSignal | undefined, timeoutMs: number, graceMs?: number);
|
|
152
|
+
/** Whether OUR hard timer fired (vs an outer cancellation). */
|
|
133
153
|
get isTimeout(): boolean;
|
|
134
|
-
/**
|
|
154
|
+
/** Whether the soft (waiting) budget has run out. */
|
|
155
|
+
get expired(): boolean;
|
|
156
|
+
/** Milliseconds left on the waiting budget, floored at 1 for Playwright options. */
|
|
135
157
|
remainingMs(): number;
|
|
136
158
|
}
|
|
137
159
|
/**
|
|
@@ -144,6 +166,10 @@ export declare class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
144
166
|
private readonly sessionIdSource;
|
|
145
167
|
readonly id = "playwright";
|
|
146
168
|
private readonly semaphore;
|
|
169
|
+
/** Waiting budget per fetch (ms); code-level, overridable by subclasses (tests). */
|
|
170
|
+
protected readonly fetchTimeoutMs: number;
|
|
171
|
+
/** Extraction grace after the waiting budget (ms); code-level, overridable by subclasses. */
|
|
172
|
+
protected readonly extractGraceMs: number;
|
|
147
173
|
/** Shared CDP connection; injectable so the suite can substitute a fake. */
|
|
148
174
|
protected readonly cdpPool: CdpConnectionPool;
|
|
149
175
|
/**
|
|
@@ -174,6 +200,17 @@ export declare class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
174
200
|
* budget before the final document is read (see the class docs).
|
|
175
201
|
*/
|
|
176
202
|
private retrieve;
|
|
203
|
+
/**
|
|
204
|
+
* The challenge dead-end's plain-HTTP refetch: one request through the
|
|
205
|
+
* page's own request context (the SAME cookie jar — any clearance the
|
|
206
|
+
* profile holds rides along — but none of Chromium's render decisions),
|
|
207
|
+
* wearing the browser's own User-Agent. Returns the finished result when
|
|
208
|
+
* the answer is a NON-challenge HTML document, undefined otherwise (the
|
|
209
|
+
* caller throws the honest WEB_FETCH_CHALLENGE). A challenge served to
|
|
210
|
+
* this request too, a non-HTML body, or any transport failure keeps the
|
|
211
|
+
* browser-only verdict.
|
|
212
|
+
*/
|
|
213
|
+
private challengeHttpFallback;
|
|
177
214
|
/**
|
|
178
215
|
* One tracked `goto`: the navigation response on success, or — when
|
|
179
216
|
* Chromium failed the navigation for a reason that is not a transport
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Learned candidate selector: page-level + candidate-level features → an
|
|
3
|
+
* estimated word-F1 per candidate (gradient-boosted regression trees trained
|
|
4
|
+
* offline by `benchmark/train-selector.py`); the pipeline returns the
|
|
5
|
+
* candidate with the highest estimate.
|
|
6
|
+
*
|
|
7
|
+
* Page features double as a page-type signal (generator meta, JSON-LD
|
|
8
|
+
* `@type`, `og:type`, URL shape, DOM statistics), so the trees learn
|
|
9
|
+
* type-conditional choices — "prune on articles, keep the link directory on
|
|
10
|
+
* docs", "unwrap noscript on forums only" — that no single global threshold
|
|
11
|
+
* could express.
|
|
12
|
+
*
|
|
13
|
+
* @module dsh-web-fetch-playwright/selector
|
|
14
|
+
*/
|
|
15
|
+
import { type CandidateKind, type GenerationFacts } from './candidates.ts';
|
|
16
|
+
import { type TreeEnsemble } from './trees.ts';
|
|
17
|
+
/** Text statistics of one sanitized candidate. */
|
|
18
|
+
export interface CandidateStats {
|
|
19
|
+
kind: CandidateKind;
|
|
20
|
+
legacy: boolean;
|
|
21
|
+
words: number;
|
|
22
|
+
linkWords: number;
|
|
23
|
+
paragraphs: number;
|
|
24
|
+
headings: number;
|
|
25
|
+
listItems: number;
|
|
26
|
+
tableCells: number;
|
|
27
|
+
pres: number;
|
|
28
|
+
images: number;
|
|
29
|
+
blocks: number;
|
|
30
|
+
tokens: Set<string>;
|
|
31
|
+
consentHits: number;
|
|
32
|
+
/** Text blocks as (key, words, linked words), for block-level diffs. */
|
|
33
|
+
blockList: {
|
|
34
|
+
key: string;
|
|
35
|
+
words: number;
|
|
36
|
+
linkWords: number;
|
|
37
|
+
}[];
|
|
38
|
+
}
|
|
39
|
+
/** Measure one sanitized candidate body. */
|
|
40
|
+
export declare function measureCandidate(kind: CandidateKind, legacy: boolean, root: Element): CandidateStats;
|
|
41
|
+
/** Page-level feature names, in vector order. */
|
|
42
|
+
export declare const PAGE_FEATURES: readonly ["body_words_log", "noscript_share", "body_link_density", "readability_null", "gate_pass", "prune_changed", "has_main", "has_preloaded", "record_items_log", "record_share", ...string[], "og_article", "og_product", "og_website", "n_article_log", "n_h1", "n_h2_log", "n_pre_log", "n_time_log", "n_form_log", "n_price_log", "n_itemprop_price", "n_product_itemtype", "n_post_class_log", "n_table_log", "n_code_log", "n_card_class_log"];
|
|
43
|
+
/** Candidate-level feature names, in vector order (one-hot kinds appended). */
|
|
44
|
+
export declare const CANDIDATE_FEATURES: readonly ["words_log", "words_vs_doc", "words_vs_max", "words_vs_art_log", "link_density", "paragraphs_log", "headings_log", "li_share", "td_log", "pre_log", "img_log", "words_per_block", "precision_vs_art", "recall_vs_art", "precision_vs_doc", "consent_hits", "legacy", "extra_share", "extra_link_density", "extra_long_log", "extra_words_per_block", "missing_share", ...string[]];
|
|
45
|
+
export declare const FEATURE_NAMES: string[];
|
|
46
|
+
/** Compute the page feature vector. */
|
|
47
|
+
export declare function pageFeatures(document: Document, url: string, facts: GenerationFacts, doc: CandidateStats | undefined, bodyText: number): number[];
|
|
48
|
+
/** Compute one candidate's feature vector (page vector excluded). */
|
|
49
|
+
export declare function candidateFeatures(stats: CandidateStats, all: CandidateStats[]): number[];
|
|
50
|
+
/** A trained gradient-boosted regressor. */
|
|
51
|
+
export interface SelectorModel extends TreeEnsemble {
|
|
52
|
+
version: string;
|
|
53
|
+
features: readonly string[];
|
|
54
|
+
/** Estimate lead another candidate needs over the legacy pick to replace it. */
|
|
55
|
+
margin?: number;
|
|
56
|
+
}
|
|
57
|
+
/** Estimated quality of one candidate; falls back to the legacy choice without a model. */
|
|
58
|
+
export declare function scoreCandidate(vector: number[], model?: SelectorModel): number;
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Flat gradient-boosted tree inference shared by the candidate selector and
|
|
3
|
+
* the block classifier (models exported from sklearn by the benchmark
|
|
4
|
+
* training scripts).
|
|
5
|
+
*
|
|
6
|
+
* @module dsh-web-fetch-playwright/trees
|
|
7
|
+
*/
|
|
8
|
+
/** One tree in flat array form (sklearn layout: leaf when left === -1). */
|
|
9
|
+
export interface FlatTree {
|
|
10
|
+
feature: number[];
|
|
11
|
+
threshold: number[];
|
|
12
|
+
left: number[];
|
|
13
|
+
right: number[];
|
|
14
|
+
value: number[];
|
|
15
|
+
}
|
|
16
|
+
/** Any additive tree ensemble: `init + learningRate * Σ tree(x)`. */
|
|
17
|
+
export interface TreeEnsemble {
|
|
18
|
+
init: number;
|
|
19
|
+
learningRate: number;
|
|
20
|
+
trees: FlatTree[];
|
|
21
|
+
/** Compare in float64 (HistGradientBoosting) instead of sklearn trees' float32. */
|
|
22
|
+
float64?: boolean;
|
|
23
|
+
}
|
|
24
|
+
/** Raw ensemble output for one feature vector. */
|
|
25
|
+
export declare function predictTrees(model: TreeEnsemble, x: number[]): number;
|
package/lib/types/types.d.ts
CHANGED
|
@@ -33,17 +33,18 @@ export interface PlaywrightDownload {
|
|
|
33
33
|
cancel(): Promise<void>;
|
|
34
34
|
}
|
|
35
35
|
/**
|
|
36
|
-
* The page's `APIRequestContext` surface the download recovery
|
|
37
|
-
* through — plain HTTP from the
|
|
38
|
-
* and none of Chromium's
|
|
39
|
-
* {@link PlaywrightResponse} because
|
|
40
|
-
* `APIResponse` is for the members this
|
|
41
|
-
* do not exercise the recovery.
|
|
36
|
+
* The page's `APIRequestContext` surface the download recovery and the
|
|
37
|
+
* challenge dead-end fallback refetch through — plain HTTP from the
|
|
38
|
+
* Playwright client, the context's cookie jar, and none of Chromium's
|
|
39
|
+
* render-or-download semantics. Reuses {@link PlaywrightResponse} because
|
|
40
|
+
* that is structurally what an `APIResponse` is for the members this
|
|
41
|
+
* plugin reads. Absent on fakes that do not exercise the recovery.
|
|
42
42
|
*/
|
|
43
43
|
export interface PlaywrightRequestContext {
|
|
44
44
|
get(url: string, options?: {
|
|
45
45
|
timeout?: number;
|
|
46
46
|
signal?: AbortSignal;
|
|
47
|
+
headers?: Record<string, string>;
|
|
47
48
|
}): Promise<PlaywrightResponse>;
|
|
48
49
|
}
|
|
49
50
|
/** The request side of a response, for main-frame filtering. */
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dsh-web-fetch-playwright",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.22",
|
|
4
4
|
"description": "Playwright/CDP web-fetch provider for DeepSeek Harness: renders pages in a real browser, denoises them (Readability + DOMPurify), and returns markdown.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|