dsh-web-fetch-playwright 0.2.19 → 0.2.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,21 +1,19 @@
1
1
  /**
2
2
  * The denoise pipeline: rendered HTML → sanitized article HTML → markdown.
3
3
  *
4
- * The classic stack, in order: jsdom parses the page Playwright rendered;
5
- * inline `data:` image payloads are elided to size placeholders (build
6
- * tools inline images as base64, which would otherwise dominate the body);
7
- * Mozilla Readability extracts the article (dropping nav bars, sidebars,
8
- * footers, and ad chrome by scoring link density and text mass) — its result
9
- * must then pass a coverage gate that distrusts tiny or sliver-thin
10
- * extractions, which on list-shaped pages (portals, catalogs) is exactly
11
- * what the link-density penalty leaves behind; an accepted extraction is
12
- * then pruned of trailing link-dense sections — the post-article
13
- * recommendation rails and share bars the scorer's vocabularies cannot name
14
- * (they arrive as bare divs with no class to match); DOMPurify sanitizes
15
- * whatever HTML remains and forbids the layout tags noise lives in;
16
- * Turndown with the GFM plugin converts to markdown using the same style
17
- * options and span-safe table rules as the shipped `dsh-tool-web` renderer,
18
- * so output is consistent with what `web_fetch` produces elsewhere.
4
+ * Since 0.2.20 the pipeline is "propose, then select": jsdom parses the
5
+ * page; inline `data:` image payloads are elided to size placeholders;
6
+ * every extraction heuristic proposes a candidate (`./candidates.ts` —
7
+ * Readability pruned/unpruned/lenient, the semantic main container, the
8
+ * whole document with and without consent/chrome stripping, the repeated
9
+ * record region, noscript-unwrapped variants, the Discourse preloaded post
10
+ * stream) plus the block classifier's pick (`./blocks.ts`); DOMPurify
11
+ * sanitizes each and forbids the layout tags noise lives in; a learned
12
+ * selector (`./selector.ts`) estimates each candidate's quality from page-
13
+ * and candidate-level features and the best one wins, with the 0.2.18
14
+ * choice kept unless another candidate clearly beats it. Turndown with the
15
+ * GFM plugin converts the winner using the same style options and span-safe
16
+ * table rules as the shipped `dsh-tool-web` renderer.
19
17
  *
20
18
  * Synchronous for its callers and unit-tested against fixture pages: jsdom
21
19
  * itself is loaded lazily on the first call — and, inside a dsh profile, only
@@ -24,6 +22,8 @@
24
22
  *
25
23
  * @module dsh-web-fetch-playwright/markdown
26
24
  */
25
+ import { type CandidateKind } from './candidates.ts';
26
+ import { type CandidateStats } from './selector.ts';
27
27
  /** Which extraction path produced a result. */
28
28
  export type DenoiseMode = 'article' | 'document';
29
29
  /** One denoise pipeline outcome. */
@@ -31,23 +31,49 @@ export interface DenoiseResult {
31
31
  /** The markdown body (title, when found, already prepended). */
32
32
  markdown: string;
33
33
  /**
34
- * `article` = Readability extraction that passed the coverage gate;
35
- * `document` = whole-document fallback (Readability returned nothing, or
36
- * the gate distrusted what it returned — list-shaped pages).
34
+ * `article` = a content-focused candidate (Readability, main container,
35
+ * record region, forum post stream); `document` = a whole-document
36
+ * candidate (sanitized body, optionally chrome-stripped).
37
37
  */
38
38
  mode: DenoiseMode;
39
+ /** The candidate strategy the selector picked. */
40
+ strategy: CandidateKind;
39
41
  }
42
+ /** One candidate after sanitization, with its selector inputs. */
43
+ export interface ScoredCandidate {
44
+ kind: CandidateKind;
45
+ title?: string;
46
+ legacy: boolean;
47
+ root: HTMLElement;
48
+ stats: CandidateStats;
49
+ vector: number[];
50
+ score: number;
51
+ }
52
+ /**
53
+ * Parse, generate, sanitize, featurize, and score every candidate.
54
+ * @returns candidates sorted by the selector's estimate, best first.
55
+ */
56
+ export declare function scoreCandidates(html: string, url: string): ScoredCandidate[];
57
+ /**
58
+ * Block-classifier training rows for one page (benchmark tooling): every
59
+ * text block's feature vector and its text, for labeling against ground
60
+ * truth.
61
+ */
62
+ export declare function blockTrainingRows(html: string, url: string): {
63
+ vector: number[];
64
+ text: string;
65
+ }[];
66
+ /** Render one sanitized candidate to markdown (title prepended when found). */
67
+ export declare function renderCandidate(candidate: Pick<ScoredCandidate, 'root' | 'title'>): string;
40
68
  /**
41
69
  * Convert one rendered HTML document to denoised markdown.
42
70
  *
43
- * Readability failure (non-article pages, or extractions the coverage gate
44
- * {@link articleCoversPage} distrusts) degrades to converting the sanitized
45
- * whole document — layout tags are still forbidden, so the fallback stays
46
- * cleaner than raw turndown, and a degraded page beats an error for a body
47
- * the browser already rendered.
71
+ * Every extraction heuristic proposes a candidate (see `./candidates.ts`);
72
+ * the learned selector (`./selector.ts`) estimates each candidate's quality
73
+ * from page- and candidate-level features and the best one is rendered.
48
74
  *
49
75
  * @param html - the rendered page HTML (`page.content()`).
50
76
  * @param url - the page URL, used to resolve relative links during parsing.
51
- * @returns the markdown and the extraction mode used.
77
+ * @returns the markdown, the extraction mode, and the chosen strategy.
52
78
  */
53
79
  export declare function htmlToMarkdown(html: string, url: string): DenoiseResult;
@@ -55,9 +55,14 @@
55
55
  * behavior). The wait tracks the LAST main-frame navigation response (the
56
56
  * real page reloads in after the challenge clears) and watches the live DOM
57
57
  * so SPA-style clears are caught too. It never clicks, never injects
58
- * answers, never touches cookies itself; when the budget runs out the fetch
59
- * fails with {@link WEB_FETCH_CHALLENGE_CODE} instead of returning the
60
- * interstitial as content.
58
+ * answers, never touches cookies itself; when the budget runs out, ONE
59
+ * plain-HTTP refetch of the same URL through the page's request context
60
+ * (same cookie jar, the browser's UA, none of Chromium's render decisions
61
+ * — `challengeHttpFallback`, default on) tries to retrieve the real
62
+ * document, gated by the same response-level and content-level challenge
63
+ * classifiers so an interstitial can never pass as content. When that too
64
+ * fails, the fetch fails with {@link WEB_FETCH_CHALLENGE_CODE} instead of
65
+ * returning the interstitial as content.
61
66
  *
62
67
  * Deep capture (viewport-virtualized feeds and app-shell documents): feed-
63
68
  * shaped pages — Discourse topic streams, comment threads, timelines —
@@ -84,6 +89,17 @@
84
89
  * degrades to the legacy capture (see `src/autoscroll.ts` for the measured
85
90
  * linux.do and Feishu badcases).
86
91
  *
92
+ * Deadline (soft, then hard): each fetch has a {@link DEFAULT_TIMEOUT_MS}
93
+ * budget for WAITING — navigation, settle, challenge wait, scroll walk. When
94
+ * it runs out the page is NOT torn down: every wait is bounded by the same
95
+ * budget and simply ends, a navigation still loading is stopped
96
+ * (`window.stop()`) once its main-frame response has arrived, and whatever
97
+ * the page holds by then goes through the normal extraction — a slow or
98
+ * endless page still hands the agent the content it already rendered
99
+ * (flagged `truncated` when the document never finished loading). Only when
100
+ * nothing arrived at all, or extraction itself hangs past
101
+ * {@link EXTRACT_GRACE_MS}, does the fetch fail with `WEB_FETCH_TIMEOUT`.
102
+ *
87
103
  * @module dsh-web-fetch-playwright/provider
88
104
  */
89
105
  import type { WebFetchProvider, WebFetchRequest, WebFetchResult } from '@deepseek-ai/dsh-web';
@@ -119,19 +135,25 @@ export interface BrowserSession {
119
135
  release?: () => Promise<void>;
120
136
  }
121
137
  /**
122
- * A single abort deadline composing the caller's signal with this provider's
123
- * time budget. The timeout reason is kept apart from an outer abort so the
124
- * error translation can pick `WEB_FETCH_TIMEOUT` over `WEB_ABORTED`.
138
+ * The fetch's two-stage deadline composed with the caller's signal. The SOFT
139
+ * stage (`timeoutMs`) only bounds waiting: {@link remainingMs} counts down to
140
+ * it and {@link expired} reports it, so every wait ends there and the fetch
141
+ * extracts what the page holds. The HARD stage (`timeoutMs + graceMs`) aborts
142
+ * {@link signal}, which closes the page. The timeout reason is kept apart
143
+ * from an outer abort so the error translation can pick `WEB_FETCH_TIMEOUT`
144
+ * over `WEB_ABORTED`.
125
145
  */
126
146
  declare class Deadline {
127
147
  readonly signal: AbortSignal;
128
148
  private readonly controller;
129
149
  private readonly expiresAt;
130
150
  private timedOut;
131
- constructor(outer: AbortSignal | undefined, timeoutMs: number);
132
- /** Whether OUR timer fired (vs an outer cancellation). */
151
+ constructor(outer: AbortSignal | undefined, timeoutMs: number, graceMs?: number);
152
+ /** Whether OUR hard timer fired (vs an outer cancellation). */
133
153
  get isTimeout(): boolean;
134
- /** Milliseconds left on the budget, floored at 1 for Playwright options. */
154
+ /** Whether the soft (waiting) budget has run out. */
155
+ get expired(): boolean;
156
+ /** Milliseconds left on the waiting budget, floored at 1 for Playwright options. */
135
157
  remainingMs(): number;
136
158
  }
137
159
  /**
@@ -144,6 +166,10 @@ export declare class PlaywrightFetchProvider implements WebFetchProvider {
144
166
  private readonly sessionIdSource;
145
167
  readonly id = "playwright";
146
168
  private readonly semaphore;
169
+ /** Waiting budget per fetch (ms); code-level, overridable by subclasses (tests). */
170
+ protected readonly fetchTimeoutMs: number;
171
+ /** Extraction grace after the waiting budget (ms); code-level, overridable by subclasses. */
172
+ protected readonly extractGraceMs: number;
147
173
  /** Shared CDP connection; injectable so the suite can substitute a fake. */
148
174
  protected readonly cdpPool: CdpConnectionPool;
149
175
  /**
@@ -174,6 +200,17 @@ export declare class PlaywrightFetchProvider implements WebFetchProvider {
174
200
  * budget before the final document is read (see the class docs).
175
201
  */
176
202
  private retrieve;
203
+ /**
204
+ * The challenge dead-end's plain-HTTP refetch: one request through the
205
+ * page's own request context (the SAME cookie jar — any clearance the
206
+ * profile holds rides along — but none of Chromium's render decisions),
207
+ * wearing the browser's own User-Agent. Returns the finished result when
208
+ * the answer is a NON-challenge HTML document, undefined otherwise (the
209
+ * caller throws the honest WEB_FETCH_CHALLENGE). A challenge served to
210
+ * this request too, a non-HTML body, or any transport failure keeps the
211
+ * browser-only verdict.
212
+ */
213
+ private challengeHttpFallback;
177
214
  /**
178
215
  * One tracked `goto`: the navigation response on success, or — when
179
216
  * Chromium failed the navigation for a reason that is not a transport
@@ -0,0 +1,2 @@
1
+ import type { SelectorModel } from './selector.ts';
2
+ export declare const SELECTOR_MODEL: SelectorModel;
@@ -0,0 +1,58 @@
1
+ /**
2
+ * Learned candidate selector: page-level + candidate-level features → an
3
+ * estimated word-F1 per candidate (gradient-boosted regression trees trained
4
+ * offline by `benchmark/train-selector.py`); the pipeline returns the
5
+ * candidate with the highest estimate.
6
+ *
7
+ * Page features double as a page-type signal (generator meta, JSON-LD
8
+ * `@type`, `og:type`, URL shape, DOM statistics), so the trees learn
9
+ * type-conditional choices — "prune on articles, keep the link directory on
10
+ * docs", "unwrap noscript on forums only" — that no single global threshold
11
+ * could express.
12
+ *
13
+ * @module dsh-web-fetch-playwright/selector
14
+ */
15
+ import { type CandidateKind, type GenerationFacts } from './candidates.ts';
16
+ import { type TreeEnsemble } from './trees.ts';
17
+ /** Text statistics of one sanitized candidate. */
18
+ export interface CandidateStats {
19
+ kind: CandidateKind;
20
+ legacy: boolean;
21
+ words: number;
22
+ linkWords: number;
23
+ paragraphs: number;
24
+ headings: number;
25
+ listItems: number;
26
+ tableCells: number;
27
+ pres: number;
28
+ images: number;
29
+ blocks: number;
30
+ tokens: Set<string>;
31
+ consentHits: number;
32
+ /** Text blocks as (key, words, linked words), for block-level diffs. */
33
+ blockList: {
34
+ key: string;
35
+ words: number;
36
+ linkWords: number;
37
+ }[];
38
+ }
39
+ /** Measure one sanitized candidate body. */
40
+ export declare function measureCandidate(kind: CandidateKind, legacy: boolean, root: Element): CandidateStats;
41
+ /** Page-level feature names, in vector order. */
42
+ export declare const PAGE_FEATURES: readonly ["body_words_log", "noscript_share", "body_link_density", "readability_null", "gate_pass", "prune_changed", "has_main", "has_preloaded", "record_items_log", "record_share", ...string[], "og_article", "og_product", "og_website", "n_article_log", "n_h1", "n_h2_log", "n_pre_log", "n_time_log", "n_form_log", "n_price_log", "n_itemprop_price", "n_product_itemtype", "n_post_class_log", "n_table_log", "n_code_log", "n_card_class_log"];
43
+ /** Candidate-level feature names, in vector order (one-hot kinds appended). */
44
+ export declare const CANDIDATE_FEATURES: readonly ["words_log", "words_vs_doc", "words_vs_max", "words_vs_art_log", "link_density", "paragraphs_log", "headings_log", "li_share", "td_log", "pre_log", "img_log", "words_per_block", "precision_vs_art", "recall_vs_art", "precision_vs_doc", "consent_hits", "legacy", "extra_share", "extra_link_density", "extra_long_log", "extra_words_per_block", "missing_share", ...string[]];
45
+ export declare const FEATURE_NAMES: string[];
46
+ /** Compute the page feature vector. */
47
+ export declare function pageFeatures(document: Document, url: string, facts: GenerationFacts, doc: CandidateStats | undefined, bodyText: number): number[];
48
+ /** Compute one candidate's feature vector (page vector excluded). */
49
+ export declare function candidateFeatures(stats: CandidateStats, all: CandidateStats[]): number[];
50
+ /** A trained gradient-boosted regressor. */
51
+ export interface SelectorModel extends TreeEnsemble {
52
+ version: string;
53
+ features: readonly string[];
54
+ /** Estimate lead another candidate needs over the legacy pick to replace it. */
55
+ margin?: number;
56
+ }
57
+ /** Estimated quality of one candidate; falls back to the legacy choice without a model. */
58
+ export declare function scoreCandidate(vector: number[], model?: SelectorModel): number;
@@ -0,0 +1,25 @@
1
+ /**
2
+ * Flat gradient-boosted tree inference shared by the candidate selector and
3
+ * the block classifier (models exported from sklearn by the benchmark
4
+ * training scripts).
5
+ *
6
+ * @module dsh-web-fetch-playwright/trees
7
+ */
8
+ /** One tree in flat array form (sklearn layout: leaf when left === -1). */
9
+ export interface FlatTree {
10
+ feature: number[];
11
+ threshold: number[];
12
+ left: number[];
13
+ right: number[];
14
+ value: number[];
15
+ }
16
+ /** Any additive tree ensemble: `init + learningRate * Σ tree(x)`. */
17
+ export interface TreeEnsemble {
18
+ init: number;
19
+ learningRate: number;
20
+ trees: FlatTree[];
21
+ /** Compare in float64 (HistGradientBoosting) instead of sklearn trees' float32. */
22
+ float64?: boolean;
23
+ }
24
+ /** Raw ensemble output for one feature vector. */
25
+ export declare function predictTrees(model: TreeEnsemble, x: number[]): number;
@@ -33,17 +33,18 @@ export interface PlaywrightDownload {
33
33
  cancel(): Promise<void>;
34
34
  }
35
35
  /**
36
- * The page's `APIRequestContext` surface the download recovery refetches
37
- * through — plain HTTP from the Playwright client, the context's cookie jar,
38
- * and none of Chromium's render-or-download semantics. Reuses
39
- * {@link PlaywrightResponse} because that is structurally what an
40
- * `APIResponse` is for the members this plugin reads. Absent on fakes that
41
- * do not exercise the recovery.
36
+ * The page's `APIRequestContext` surface the download recovery and the
37
+ * challenge dead-end fallback refetch through — plain HTTP from the
38
+ * Playwright client, the context's cookie jar, and none of Chromium's
39
+ * render-or-download semantics. Reuses {@link PlaywrightResponse} because
40
+ * that is structurally what an `APIResponse` is for the members this
41
+ * plugin reads. Absent on fakes that do not exercise the recovery.
42
42
  */
43
43
  export interface PlaywrightRequestContext {
44
44
  get(url: string, options?: {
45
45
  timeout?: number;
46
46
  signal?: AbortSignal;
47
+ headers?: Record<string, string>;
47
48
  }): Promise<PlaywrightResponse>;
48
49
  }
49
50
  /** The request side of a response, for main-frame filtering. */
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "dsh-web-fetch-playwright",
3
- "version": "0.2.19",
3
+ "version": "0.2.22",
4
4
  "description": "Playwright/CDP web-fetch provider for DeepSeek Harness: renders pages in a real browser, denoises them (Readability + DOMPurify), and returns markdown.",
5
5
  "type": "module",
6
6
  "license": "MIT",