dsh-web-fetch-playwright 0.2.22 → 0.2.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/README.zh-CN.md +4 -2
- package/lib/client.js +19 -19
- package/lib/index.js +386 -50
- package/lib/types/candidates.d.ts +48 -0
- package/lib/types/challenge.d.ts +2 -2
- package/lib/types/config.d.ts +30 -4
- package/lib/types/index.d.ts +1 -1
- package/lib/types/provider.d.ts +55 -0
- package/lib/types/types.d.ts +19 -0
- package/package.json +1 -1
- package/src/candidates.ts +162 -29
- package/src/challenge.ts +2 -2
- package/src/config.ts +37 -6
- package/src/index.ts +2 -0
- package/src/markdown.ts +13 -2
- package/src/provider.ts +292 -27
- package/src/types.ts +20 -0
|
@@ -38,6 +38,15 @@ export interface RawCandidate {
|
|
|
38
38
|
kind: CandidateKind;
|
|
39
39
|
/** HTML of the extraction; sanitized by the caller. */
|
|
40
40
|
html: string;
|
|
41
|
+
/**
|
|
42
|
+
* The extraction as a live DOM node, when the generator already holds one
|
|
43
|
+
* (`doc`/`docc`/`ns_doc` serialize `body`, `main` the container itself).
|
|
44
|
+
* DOMPurify sanitizes a node by importing a copy — the caller's re-parse
|
|
45
|
+
* of `html` (hundreds of ms on large pages, measured on the Angular docs
|
|
46
|
+
* badcase: 555ms for the whole-body candidate alone) is skipped; `html`
|
|
47
|
+
* stays the identity for the dedup below.
|
|
48
|
+
*/
|
|
49
|
+
node?: Element;
|
|
41
50
|
/** Title to prepend when the body does not open with a heading. */
|
|
42
51
|
title?: string;
|
|
43
52
|
/** Whether the 0.2.18 pipeline would have returned exactly this candidate. */
|
|
@@ -153,7 +162,46 @@ export interface GenerationFacts {
|
|
|
153
162
|
hasPreloaded: boolean;
|
|
154
163
|
/** The semantic main container, when found. */
|
|
155
164
|
main: Element | null;
|
|
165
|
+
/**
|
|
166
|
+
* True when the Readability family was SKIPPED because nearly all body
|
|
167
|
+
* text is custom-wrapped ({@link customWrappedTextShare}) — every
|
|
168
|
+
* extraction that would carry it sanitizes to nothing. The orchestrator
|
|
169
|
+
* then seeds the selector with the empty-`art` reference stats such a run
|
|
170
|
+
* would have measured, keeping the surviving candidates' features
|
|
171
|
+
* (`words_vs_art_log`, the block diffs) exactly what a dead Readability
|
|
172
|
+
* pick yields.
|
|
173
|
+
*/
|
|
174
|
+
readabilitySkipped?: boolean;
|
|
156
175
|
}
|
|
176
|
+
/**
|
|
177
|
+
* At or above this share of custom-wrapped body text the Readability family
|
|
178
|
+
* is skipped (see {@link customWrappedTextShare}).
|
|
179
|
+
*/
|
|
180
|
+
export declare const CUSTOM_WRAP_SKIP_SHARE = 0.9;
|
|
181
|
+
/**
|
|
182
|
+
* Share of the body's visible text that sits inside custom-element subtrees
|
|
183
|
+
* (tag names containing `-`: `<ms-app>`, `<my-doc>` — the HTML spec reserves
|
|
184
|
+
* them for custom elements; no standard HTML or MathML tag and no tag the
|
|
185
|
+
* denoise pipeline's DOMPurify step keeps contains a dash). DOMPurify's
|
|
186
|
+
* allowlist removes such tags wholesale — `KEEP_CONTENT: false` takes their
|
|
187
|
+
* subtrees with them — so once nearly ALL the page's text is custom-wrapped,
|
|
188
|
+
* every extraction that would carry it sanitizes to a corpse. Measured on
|
|
189
|
+
* the badcase (aistudio.google.com/docs/computer-use, an Angular app whose
|
|
190
|
+
* entire DOM lives under `<ms-app>`): 99% of body text custom-wrapped;
|
|
191
|
+
* Readability's pick sanitized to a root with ZERO words, the whole-document
|
|
192
|
+
* candidate to 39 of ~4.5k words — while the three Readability passes over
|
|
193
|
+
* the 1.1MB document cost ~6.5s of pure latency on candidates that could
|
|
194
|
+
* never win. The DOM-backed fallbacks, the noscript paths and the block
|
|
195
|
+
* classifier — the candidates that survive DOMPurify on this shape, and the
|
|
196
|
+
* ones the selector picks on such pages anyway — carry the extraction.
|
|
197
|
+
*
|
|
198
|
+
* Noise subtrees (script/style/nav/…) count as neither wrapped nor unwrapped;
|
|
199
|
+
* the share is over CONTENT-bearing text only. A partial share (a web
|
|
200
|
+
* component widget inside a normal article) stays below the threshold and
|
|
201
|
+
* keeps the full pipeline: only the shape where the family is provably dead
|
|
202
|
+
* pays for it.
|
|
203
|
+
*/
|
|
204
|
+
export declare function customWrappedTextShare(body: Element): number;
|
|
157
205
|
/**
|
|
158
206
|
* Propose every candidate extraction for one parsed page.
|
|
159
207
|
* @param document - the parsed page (data URIs already elided); not mutated.
|
package/lib/types/challenge.d.ts
CHANGED
|
@@ -41,8 +41,8 @@
|
|
|
41
41
|
export declare const CHALLENGE_POLL_INTERVAL_MS = 500;
|
|
42
42
|
/**
|
|
43
43
|
* Tail of the fetch budget held back from the challenge wait so a cleared
|
|
44
|
-
* challenge still has time for its settle (
|
|
45
|
-
* content read and the denoise pipeline.
|
|
44
|
+
* challenge still has time for its settle (the `DEFAULT_SETTLE_MS` cap)
|
|
45
|
+
* plus the content read and the denoise pipeline.
|
|
46
46
|
*/
|
|
47
47
|
export declare const CHALLENGE_FINISH_RESERVE_MS = 8000;
|
|
48
48
|
/**
|
package/lib/types/config.d.ts
CHANGED
|
@@ -25,6 +25,14 @@ import z from '@deepseek-ai/schemastery';
|
|
|
25
25
|
* skipped outright.
|
|
26
26
|
*/
|
|
27
27
|
export declare const DEFAULT_SCROLL_BUDGET_MS = 20000;
|
|
28
|
+
/**
|
|
29
|
+
* Default cap (ms) on the post-navigation settle — the wait for SPA
|
|
30
|
+
* rendering to finish once the document has loaded (see the settle in
|
|
31
|
+
* `src/provider.ts`). A CAP, not a fixed cost: the settle ends as soon as
|
|
32
|
+
* the page's network goes quiet. Code-level only (tests, demo scripts):
|
|
33
|
+
* not part of the settings schema. `0` disables the settle entirely.
|
|
34
|
+
*/
|
|
35
|
+
export declare const DEFAULT_SETTLE_MS = 5000;
|
|
28
36
|
/** Default CDP endpoint when the settings section leaves it blank. */
|
|
29
37
|
export declare const DEFAULT_CDP_ENDPOINT = "127.0.0.1:9222";
|
|
30
38
|
/**
|
|
@@ -135,6 +143,14 @@ export interface Config {
|
|
|
135
143
|
* configure it. Default: {@link DEFAULT_SCROLL_BUDGET_MS}.
|
|
136
144
|
*/
|
|
137
145
|
scrollBudgetMs?: number;
|
|
146
|
+
/**
|
|
147
|
+
* Cap (ms) on the post-navigation settle (load baseline + bounded
|
|
148
|
+
* network-quiet window with stale-request discount). Code-level only
|
|
149
|
+
* (tests, demo scripts): not part of the settings schema, so users cannot
|
|
150
|
+
* configure it. `0` disables the settle. Default:
|
|
151
|
+
* {@link DEFAULT_SETTLE_MS}.
|
|
152
|
+
*/
|
|
153
|
+
settleMs?: number;
|
|
138
154
|
/**
|
|
139
155
|
* How many fetches may render at once. Blank = backend default (4 for
|
|
140
156
|
* local — each slot launches a browser; 50 for CDP — each slot is a tab in
|
|
@@ -214,14 +230,15 @@ export declare function resolveConfig(config: Partial<Config>): ResolvedConfig;
|
|
|
214
230
|
/**
|
|
215
231
|
* Complete config after schemastery applies the field defaults it owns.
|
|
216
232
|
* `maxConcurrency` stays optional because its effective default depends on
|
|
217
|
-
* `backend`; `challengeWaitMs` and `
|
|
218
|
-
* they are code-level overrides outside the schema —
|
|
219
|
-
* effective* helpers supply the built-in defaults.
|
|
233
|
+
* `backend`; `challengeWaitMs`, `scrollBudgetMs` and `settleMs` stay
|
|
234
|
+
* optional because they are code-level overrides outside the schema —
|
|
235
|
+
* absent, the effective* helpers supply the built-in defaults.
|
|
220
236
|
*/
|
|
221
|
-
export type ResolvedConfig = Omit<Required<Config>, 'maxConcurrency' | 'challengeWaitMs' | 'scrollBudgetMs'> & {
|
|
237
|
+
export type ResolvedConfig = Omit<Required<Config>, 'maxConcurrency' | 'challengeWaitMs' | 'scrollBudgetMs' | 'settleMs'> & {
|
|
222
238
|
maxConcurrency?: number;
|
|
223
239
|
challengeWaitMs?: number;
|
|
224
240
|
scrollBudgetMs?: number;
|
|
241
|
+
settleMs?: number;
|
|
225
242
|
};
|
|
226
243
|
/**
|
|
227
244
|
* The concurrency limit a fetch actually runs with: an explicit setting
|
|
@@ -264,6 +281,15 @@ export declare function effectiveChallengeHttpFallback(config: Pick<Config, 'cha
|
|
|
264
281
|
* @returns the bounded walk budget in milliseconds.
|
|
265
282
|
*/
|
|
266
283
|
export declare function effectiveScrollBudgetMs(config: Pick<Config, 'scrollBudgetMs'>): number;
|
|
284
|
+
/**
|
|
285
|
+
* The settle cap a fetch actually runs with: an explicit value wins, else
|
|
286
|
+
* the built-in default. `0` means the settle is off (domcontentloaded
|
|
287
|
+
* content is read as-is).
|
|
288
|
+
*
|
|
289
|
+
* @param config - the resolved settings section (or any partial of it).
|
|
290
|
+
* @returns the bounded settle cap in milliseconds.
|
|
291
|
+
*/
|
|
292
|
+
export declare function effectiveSettleMs(config: Pick<Config, 'settleMs'>): number;
|
|
267
293
|
/**
|
|
268
294
|
* The context mode a fetch actually runs with: profile (a tab in the remote
|
|
269
295
|
* browser's real profile) requires the CDP backend — a local launch has no
|
package/lib/types/index.d.ts
CHANGED
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
*/
|
|
16
16
|
import type { Context } from '@deepseek-ai/cordis';
|
|
17
17
|
import type { PlaywrightPluginConfig } from './config.ts';
|
|
18
|
-
export { Config, DEFAULT_CDP_ENDPOINT, DEFAULT_CHALLENGE_HTTP_FALLBACK, DEFAULT_CHALLENGE_RETRIES, DEFAULT_CHALLENGE_WAIT_MS, DEFAULT_SCROLL_BUDGET_MS, effectiveChallengeHttpFallback, effectiveChallengeRetries, effectiveChallengeWaitMs, effectiveContextMode, effectiveScrollBudgetMs, normalizeCdpEndpoint, resolveConfig, snapshotsOf, } from './config.ts';
|
|
18
|
+
export { Config, DEFAULT_CDP_ENDPOINT, DEFAULT_CHALLENGE_HTTP_FALLBACK, DEFAULT_CHALLENGE_RETRIES, DEFAULT_CHALLENGE_WAIT_MS, DEFAULT_SCROLL_BUDGET_MS, DEFAULT_SETTLE_MS, effectiveChallengeHttpFallback, effectiveChallengeRetries, effectiveChallengeWaitMs, effectiveContextMode, effectiveScrollBudgetMs, effectiveSettleMs, normalizeCdpEndpoint, resolveConfig, snapshotsOf, } from './config.ts';
|
|
19
19
|
export type { Config as PlaywrightFetchConfig, CdpContextMode, ContextIsolationScope, PlaywrightBackend, PlaywrightPluginConfig, ResolvedConfig, } from './config.ts';
|
|
20
20
|
export { CdpConnectionPool } from './cdp-pool.ts';
|
|
21
21
|
export type { CdpAcquireMode, CdpConnect, CdpLease } from './cdp-pool.ts';
|
package/lib/types/provider.d.ts
CHANGED
|
@@ -18,6 +18,21 @@
|
|
|
18
18
|
* refetch through the page's API request context
|
|
19
19
|
* ({@link recoverDownloadResponse}) — instead of a `WEB_PROVIDER_ERROR`.
|
|
20
20
|
*
|
|
21
|
+
* Transient transport failures (issue: `net::ERR_NETWORK_CHANGED` and its
|
|
22
|
+
* family) are retried, not terminal: Chromium's NetworkChangeNotifier kills
|
|
23
|
+
* every in-flight request with `ERR_NETWORK_CHANGED` the moment the HOST's
|
|
24
|
+
* network configuration shifts under the browser — a route/interface/DNS/
|
|
25
|
+
* proxy change, a VPN or DHCP flap, a Docker embedded-DNS rewrite; the page
|
|
26
|
+
* and its target are perfectly fine, only that one navigation died, and the
|
|
27
|
+
* very next `goto` usually succeeds (measured live: source.android.com
|
|
28
|
+
* failed once mid-navigation in one DSH session and fetched clean from the
|
|
29
|
+
* same host seconds later). Each navigation therefore retries the
|
|
30
|
+
* transport-blip family ({@link isTransientNavigationFailure} — deliberately
|
|
31
|
+
* NOT `ERR_ABORTED`, which is overloaded with the deterministic shapes the
|
|
32
|
+
* recoveries above already own) a bounded number of times with a short
|
|
33
|
+
* backoff, inside the same page and the remaining waiting budget; a blip
|
|
34
|
+
* that survives the attempt cap still fails as before.
|
|
35
|
+
*
|
|
21
36
|
* Lifecycle: the local backend launches a browser per fetch and closes it —
|
|
22
37
|
* nothing outlives the call. CDP session isolation keeps one connection per
|
|
23
38
|
* DSH session until its context becomes idle; this also isolates gateways that
|
|
@@ -170,6 +185,12 @@ export declare class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
170
185
|
protected readonly fetchTimeoutMs: number;
|
|
171
186
|
/** Extraction grace after the waiting budget (ms); code-level, overridable by subclasses. */
|
|
172
187
|
protected readonly extractGraceMs: number;
|
|
188
|
+
/** Settle quiet window (ms); code-level, overridable by subclasses (tests). */
|
|
189
|
+
protected readonly settleQuietMs: number;
|
|
190
|
+
/** Request age (ms) past which it stops gating the settle; overridable by subclasses (tests). */
|
|
191
|
+
protected readonly settleStaleMs: number;
|
|
192
|
+
/** Settle re-check cadence (ms); overridable by subclasses (tests). */
|
|
193
|
+
protected readonly settlePollMs: number;
|
|
173
194
|
/** Shared CDP connection; injectable so the suite can substitute a fake. */
|
|
174
195
|
protected readonly cdpPool: CdpConnectionPool;
|
|
175
196
|
/**
|
|
@@ -219,6 +240,11 @@ export declare class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
219
240
|
* status-only 204/205 — {@link recoverUnrenderedStatus}), and the response
|
|
220
241
|
* being a download the browser refuses to commit as a document
|
|
221
242
|
* (`Content-Disposition: attachment` — {@link recoverDownloadResponse}).
|
|
243
|
+
* A TRANSIENT transport failure (`net::ERR_NETWORK_CHANGED` and family)
|
|
244
|
+
* is neither: it carries no response and no render decision, only a
|
|
245
|
+
* momentary host-network blip that killed this one navigation — so it is
|
|
246
|
+
* retried on the SAME page, bounded by {@link MAX_TRANSIENT_NAV_RETRIES}
|
|
247
|
+
* attempts, a short backoff, and the remaining waiting budget.
|
|
222
248
|
*
|
|
223
249
|
* @returns the response `goto` resolved with, plus `recovered` when the
|
|
224
250
|
* navigation failure was recovered into a finished result the caller must
|
|
@@ -244,5 +270,34 @@ export declare class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
244
270
|
* @returns true when the challenge cleared; false when the budget ran out.
|
|
245
271
|
*/
|
|
246
272
|
private waitForChallengeClear;
|
|
273
|
+
/**
|
|
274
|
+
* Best-effort settle for client-rendered content — a Lighthouse-style
|
|
275
|
+
* bounded network-quiet window replacing Playwright's `networkidle`
|
|
276
|
+
* (officially discouraged): worker-script fetches and cross-origin auth
|
|
277
|
+
* frames hold one request open on the page session FOREVER (their
|
|
278
|
+
* completion events are attributed to other CDP targets), so strict
|
|
279
|
+
* networkidle never arrives on exactly the modern pages the settle exists
|
|
280
|
+
* for, and the legacy wait burned its whole cap on them every fetch.
|
|
281
|
+
*
|
|
282
|
+
* Sequence: a `load` baseline (usually already past — domcontentloaded
|
|
283
|
+
* resolved earlier), then poll until no request has STARTED within
|
|
284
|
+
* {@link settleQuietMs} and every still-open request is older than
|
|
285
|
+
* {@link settleStaleMs} — a request that old is an attribution artifact
|
|
286
|
+
* or a long-poll whose completion no longer gates content. All waits
|
|
287
|
+
* share the {@link effectiveSettleMs} cap and the fetch deadline; a page
|
|
288
|
+
* that exposes no request events degrades to the legacy bounded
|
|
289
|
+
* networkidle wait. Never throws: any exit path keeps what the document
|
|
290
|
+
* already holds.
|
|
291
|
+
*/
|
|
292
|
+
private settleForContent;
|
|
293
|
+
/**
|
|
294
|
+
* Settle outcome observation hook — silent in production by design (the
|
|
295
|
+
* settle is best-effort), but subclasses (tests, diagnostics) override it
|
|
296
|
+
* to see which exit path every fetch took: `off` (config), `quiet`
|
|
297
|
+
* (network went quiet), `stale-discounted` (only stale requests remained
|
|
298
|
+
* open), `cap` (window exhausted), `fallback` (no request events — the
|
|
299
|
+
* legacy networkidle wait ran), `aborted` (fetch deadline fired).
|
|
300
|
+
*/
|
|
301
|
+
protected noteSettleOutcome(outcome: 'off' | 'quiet' | 'stale-discounted' | 'cap' | 'fallback' | 'aborted', elapsedMs: number): void;
|
|
247
302
|
}
|
|
248
303
|
export {};
|
package/lib/types/types.d.ts
CHANGED
|
@@ -56,6 +56,14 @@ export interface PlaywrightRequest {
|
|
|
56
56
|
/** The frame that issued the request; compare with `page.mainFrame()`. */
|
|
57
57
|
frame?(): unknown;
|
|
58
58
|
}
|
|
59
|
+
/**
|
|
60
|
+
* A network request notification, as `page.on('request')` and its settled
|
|
61
|
+
* siblings deliver it. Identity only — the settle's network-quiet tracker
|
|
62
|
+
* keys its open-request map on the object itself.
|
|
63
|
+
*/
|
|
64
|
+
export interface PlaywrightWebRequest {
|
|
65
|
+
url?(): string;
|
|
66
|
+
}
|
|
59
67
|
/** A page inside a context. */
|
|
60
68
|
export interface PlaywrightPage {
|
|
61
69
|
goto(url: string, options?: {
|
|
@@ -81,6 +89,17 @@ export interface PlaywrightPage {
|
|
|
81
89
|
* document). Absent on minimal fakes (content polling covers them).
|
|
82
90
|
*/
|
|
83
91
|
on?(event: 'response', listener: (response: PlaywrightResponse) => void): unknown;
|
|
92
|
+
/**
|
|
93
|
+
* Request-start notification — the settle's network-quiet tracker. Absent
|
|
94
|
+
* on minimal fakes (the settle falls back to the legacy bounded
|
|
95
|
+
* networkidle wait).
|
|
96
|
+
*/
|
|
97
|
+
on?(event: 'request', listener: (request: PlaywrightWebRequest) => void): unknown;
|
|
98
|
+
/**
|
|
99
|
+
* Request-settled notification (finished or failed) — closes the tracker's
|
|
100
|
+
* open-request bookkeeping. Absent on minimal fakes.
|
|
101
|
+
*/
|
|
102
|
+
on?(event: 'requestfinished' | 'requestfailed', listener: (request: PlaywrightWebRequest) => void): unknown;
|
|
84
103
|
/**
|
|
85
104
|
* Download notification — a navigation the browser handed to its download
|
|
86
105
|
* manager instead of committing a document (`Content-Disposition:
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dsh-web-fetch-playwright",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.23",
|
|
4
4
|
"description": "Playwright/CDP web-fetch provider for DeepSeek Harness: renders pages in a real browser, denoises them (Readability + DOMPurify), and returns markdown.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
package/src/candidates.ts
CHANGED
|
@@ -42,6 +42,15 @@ export interface RawCandidate {
|
|
|
42
42
|
kind: CandidateKind
|
|
43
43
|
/** HTML of the extraction; sanitized by the caller. */
|
|
44
44
|
html: string
|
|
45
|
+
/**
|
|
46
|
+
* The extraction as a live DOM node, when the generator already holds one
|
|
47
|
+
* (`doc`/`docc`/`ns_doc` serialize `body`, `main` the container itself).
|
|
48
|
+
* DOMPurify sanitizes a node by importing a copy — the caller's re-parse
|
|
49
|
+
* of `html` (hundreds of ms on large pages, measured on the Angular docs
|
|
50
|
+
* badcase: 555ms for the whole-body candidate alone) is skipped; `html`
|
|
51
|
+
* stays the identity for the dedup below.
|
|
52
|
+
*/
|
|
53
|
+
node?: Element
|
|
45
54
|
/** Title to prepend when the body does not open with a heading. */
|
|
46
55
|
title?: string
|
|
47
56
|
/** Whether the 0.2.18 pipeline would have returned exactly this candidate. */
|
|
@@ -113,8 +122,20 @@ const GATE_MIN_ARTICLE_COVERAGE = 0.1
|
|
|
113
122
|
export function articleCoversPage(articleHtml: string, document: Document): boolean {
|
|
114
123
|
const bodyText = visibleTextLength(document.body)
|
|
115
124
|
if (bodyText < GATE_MIN_BODY_TEXT_CHARS) return true
|
|
116
|
-
|
|
117
|
-
|
|
125
|
+
return articleCoversParsed(htmlFragment(articleHtml, document), bodyText)
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* {@link articleCoversPage} on an already-parsed article container, so the
|
|
130
|
+
* generator's gate and trailing prune share ONE parse of the article — the
|
|
131
|
+
* extraction is hundreds of KB on heavy pages (measured on the Angular docs
|
|
132
|
+
* badcase: ~400ms per parse of Readability's 485KB pick).
|
|
133
|
+
*
|
|
134
|
+
* @param container - the parsed article (read, never mutated).
|
|
135
|
+
* @param bodyText - the page body's visible text length, computed once.
|
|
136
|
+
*/
|
|
137
|
+
function articleCoversParsed(container: Element, bodyText: number): boolean {
|
|
138
|
+
if (bodyText < GATE_MIN_BODY_TEXT_CHARS) return true
|
|
118
139
|
const articleText = visibleTextLength(container)
|
|
119
140
|
return articleText >= GATE_MIN_ARTICLE_TEXT_CHARS
|
|
120
141
|
&& articleText / bodyText >= GATE_MIN_ARTICLE_COVERAGE
|
|
@@ -190,8 +211,19 @@ function removeTrailingLinkDenseRun(parent: Element, articleText: number): void
|
|
|
190
211
|
|
|
191
212
|
/** Prune with the 0.2.18 misfire guard; returns the original on a misfire. */
|
|
192
213
|
export function prunedArticleContent(articleHtml: string, document: Document): string {
|
|
193
|
-
|
|
194
|
-
|
|
214
|
+
return pruneParsedArticle(htmlFragment(articleHtml, document), articleHtml)
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
/**
|
|
218
|
+
* {@link prunedArticleContent} on an already-parsed article container (the
|
|
219
|
+
* generator passes the gate's parse: the gate reads it first, then this
|
|
220
|
+
* mutates it — the same order the two-string-parses version produced).
|
|
221
|
+
*
|
|
222
|
+
* @param container - the parsed article; pruned in place.
|
|
223
|
+
* @param articleHtml - the string the container was parsed from (the
|
|
224
|
+
* misfire-guard's verbatim return).
|
|
225
|
+
*/
|
|
226
|
+
function pruneParsedArticle(container: Element, articleHtml: string): string {
|
|
195
227
|
const before = visibleTextLength(container)
|
|
196
228
|
if (before === 0) return articleHtml
|
|
197
229
|
pruneTrailingLinkDenseSections(container)
|
|
@@ -204,13 +236,25 @@ export function prunedArticleContent(articleHtml: string, document: Document): s
|
|
|
204
236
|
|
|
205
237
|
// ------------------------------------------------- recommendation rails ----
|
|
206
238
|
|
|
207
|
-
/**
|
|
239
|
+
/**
|
|
240
|
+
* Link density at or above which a recommendation-named subtree is a rail.
|
|
241
|
+
*/
|
|
208
242
|
const RAIL_LINK_DENSITY = 0.3
|
|
209
243
|
const RAIL_MIN_LINKS = 4
|
|
210
244
|
/** Prose (low-link text) the page must keep outside the rails. */
|
|
211
245
|
const RAIL_MIN_PROSE_CHARS = 80
|
|
212
246
|
const RAIL_PROSE_LINK_DENSITY = 0.2
|
|
213
247
|
|
|
248
|
+
/**
|
|
249
|
+
* Necessary condition for {@link isRailName} on the raw class+id string: a
|
|
250
|
+
* matching token (`recommended-posts`, `Related`, …) always leaves one of
|
|
251
|
+
* these literals as a substring. Testing it first short-circuits the
|
|
252
|
+
* per-token split on the pages where it matters — {@link markRecommendationRails}
|
|
253
|
+
* walks every element, and on the measured 1.1MB Angular docs badcase the
|
|
254
|
+
* split alone cost ~360ms of the candidate generation.
|
|
255
|
+
*/
|
|
256
|
+
const RAIL_NAME_HINT = /recommend|related|outbrain|taboola/
|
|
257
|
+
|
|
214
258
|
/**
|
|
215
259
|
* Class/id sub-tokens naming a recommendation / related-content rail: the
|
|
216
260
|
* English words every CMS and theme uses, plus the two syndicated
|
|
@@ -225,6 +269,7 @@ function isRailName(token: string): boolean {
|
|
|
225
269
|
|
|
226
270
|
function isRecommendationRail(element: Element): boolean {
|
|
227
271
|
const names = `${element.getAttribute('class') ?? ''} ${element.id}`.toLowerCase()
|
|
272
|
+
if (!RAIL_NAME_HINT.test(names)) return false
|
|
228
273
|
if (!names.split(/[\s_-]+/).some(isRailName)) return false
|
|
229
274
|
if (element.querySelectorAll('a').length < RAIL_MIN_LINKS) return false
|
|
230
275
|
const text = visibleTextLength(element)
|
|
@@ -562,11 +607,82 @@ export interface GenerationFacts {
|
|
|
562
607
|
hasPreloaded: boolean
|
|
563
608
|
/** The semantic main container, when found. */
|
|
564
609
|
main: Element | null
|
|
610
|
+
/**
|
|
611
|
+
* True when the Readability family was SKIPPED because nearly all body
|
|
612
|
+
* text is custom-wrapped ({@link customWrappedTextShare}) — every
|
|
613
|
+
* extraction that would carry it sanitizes to nothing. The orchestrator
|
|
614
|
+
* then seeds the selector with the empty-`art` reference stats such a run
|
|
615
|
+
* would have measured, keeping the surviving candidates' features
|
|
616
|
+
* (`words_vs_art_log`, the block diffs) exactly what a dead Readability
|
|
617
|
+
* pick yields.
|
|
618
|
+
*/
|
|
619
|
+
readabilitySkipped?: boolean
|
|
565
620
|
}
|
|
566
621
|
|
|
567
622
|
/** Below this much noscript text the noscript-unwrapped paths are skipped. */
|
|
568
623
|
const NOSCRIPT_MIN_TEXT_CHARS = 200
|
|
569
624
|
|
|
625
|
+
/**
|
|
626
|
+
* At or above this share of custom-wrapped body text the Readability family
|
|
627
|
+
* is skipped (see {@link customWrappedTextShare}).
|
|
628
|
+
*/
|
|
629
|
+
export const CUSTOM_WRAP_SKIP_SHARE = 0.9
|
|
630
|
+
|
|
631
|
+
/**
|
|
632
|
+
* Share of the body's visible text that sits inside custom-element subtrees
|
|
633
|
+
* (tag names containing `-`: `<ms-app>`, `<my-doc>` — the HTML spec reserves
|
|
634
|
+
* them for custom elements; no standard HTML or MathML tag and no tag the
|
|
635
|
+
* denoise pipeline's DOMPurify step keeps contains a dash). DOMPurify's
|
|
636
|
+
* allowlist removes such tags wholesale — `KEEP_CONTENT: false` takes their
|
|
637
|
+
* subtrees with them — so once nearly ALL the page's text is custom-wrapped,
|
|
638
|
+
* every extraction that would carry it sanitizes to a corpse. Measured on
|
|
639
|
+
* the badcase (aistudio.google.com/docs/computer-use, an Angular app whose
|
|
640
|
+
* entire DOM lives under `<ms-app>`): 99% of body text custom-wrapped;
|
|
641
|
+
* Readability's pick sanitized to a root with ZERO words, the whole-document
|
|
642
|
+
* candidate to 39 of ~4.5k words — while the three Readability passes over
|
|
643
|
+
* the 1.1MB document cost ~6.5s of pure latency on candidates that could
|
|
644
|
+
* never win. The DOM-backed fallbacks, the noscript paths and the block
|
|
645
|
+
* classifier — the candidates that survive DOMPurify on this shape, and the
|
|
646
|
+
* ones the selector picks on such pages anyway — carry the extraction.
|
|
647
|
+
*
|
|
648
|
+
* Noise subtrees (script/style/nav/…) count as neither wrapped nor unwrapped;
|
|
649
|
+
* the share is over CONTENT-bearing text only. A partial share (a web
|
|
650
|
+
* component widget inside a normal article) stays below the threshold and
|
|
651
|
+
* keeps the full pipeline: only the shape where the family is provably dead
|
|
652
|
+
* pays for it.
|
|
653
|
+
*/
|
|
654
|
+
export function customWrappedTextShare(body: Element): number {
|
|
655
|
+
let total = 0
|
|
656
|
+
let wrapped = 0
|
|
657
|
+
const visit = (node: Node, insideCustom: boolean): void => {
|
|
658
|
+
if (node.nodeType === 3) {
|
|
659
|
+
const length = (node.nodeValue ?? '').replace(/\s+/g, '').length
|
|
660
|
+
if (length === 0) return
|
|
661
|
+
total += length
|
|
662
|
+
if (insideCustom) wrapped += length
|
|
663
|
+
return
|
|
664
|
+
}
|
|
665
|
+
if (node.nodeType !== 1 || NOISE_SUBTREE_TAGS.has(node.nodeName.toLowerCase())) return
|
|
666
|
+
const custom = insideCustom || node.nodeName.includes('-')
|
|
667
|
+
for (let child = node.firstChild; child !== null; child = child.nextSibling) visit(child, custom)
|
|
668
|
+
}
|
|
669
|
+
visit(body, false)
|
|
670
|
+
return total === 0 ? 0 : wrapped / total
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
/**
|
|
674
|
+
* The title the gated path hands its candidates: Readability's own first
|
|
675
|
+
* source is `og:title`, its second the `<title>` element — both remain
|
|
676
|
+
* readable on custom-element pages, where the DOM wrapping is the framework's
|
|
677
|
+
* but the metadata is the page's.
|
|
678
|
+
*/
|
|
679
|
+
function fallbackTitle(document: Document): string | undefined {
|
|
680
|
+
const og = document.querySelector('meta[property="og:title"]')?.getAttribute('content')?.trim()
|
|
681
|
+
if (og !== undefined && og !== '') return og
|
|
682
|
+
const title = document.querySelector('title')?.textContent?.trim()
|
|
683
|
+
return title === undefined || title === '' ? undefined : title
|
|
684
|
+
}
|
|
685
|
+
|
|
570
686
|
/**
|
|
571
687
|
* Propose every candidate extraction for one parsed page.
|
|
572
688
|
* @param document - the parsed page (data URIs already elided); not mutated.
|
|
@@ -576,38 +692,54 @@ export function generateCandidates(document: Document): { candidates: RawCandida
|
|
|
576
692
|
const body = document.body
|
|
577
693
|
const bodyHtml = body?.innerHTML ?? ''
|
|
578
694
|
|
|
579
|
-
//
|
|
580
|
-
//
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
const gatePass = art !== null && articleCoversPage(art.content, document)
|
|
695
|
+
// The custom-element shape: skip the Readability family outright (see
|
|
696
|
+
// {@link customWrappedTextShare}) — its candidates sanitize to corpses.
|
|
697
|
+
const skipReadability = body !== null && customWrappedTextShare(body) >= CUSTOM_WRAP_SKIP_SHARE
|
|
698
|
+
|
|
699
|
+
let art: ReadabilityPick | null = null
|
|
700
|
+
let lenient: ReadabilityPick | null = null
|
|
701
|
+
let gatePass = false
|
|
587
702
|
let pruneChanged = false
|
|
588
|
-
if (
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
703
|
+
if (!skipReadability) {
|
|
704
|
+
// Rails are marked before Readability (it drops the class names that
|
|
705
|
+
// identify them) and cut from its output; the gate and `artu` see the
|
|
706
|
+
// extraction exactly as unmarked, so the legacy marker and the "keep
|
|
707
|
+
// everything" alternative are unchanged.
|
|
708
|
+
const marked = runReadability(markRecommendationRails(document) ?? document)
|
|
709
|
+
const rails = marked === null ? null : splitRails(marked.content, document)
|
|
710
|
+
art = marked === null || rails === null ? null : { ...marked, content: rails.plain }
|
|
711
|
+
if (art !== null) {
|
|
712
|
+
// One parse of the article serves the gate AND the prune (the common
|
|
713
|
+
// case: no rail was cut, so both read the same string).
|
|
714
|
+
const bodyText = visibleTextLength(body)
|
|
715
|
+
const parsed = htmlFragment(art.content, document)
|
|
716
|
+
gatePass = articleCoversParsed(parsed, bodyText)
|
|
717
|
+
const pruning = rails?.derailed ?? art.content
|
|
718
|
+
const pruned = pruning === art.content
|
|
719
|
+
? pruneParsedArticle(parsed, art.content)
|
|
720
|
+
: prunedArticleContent(pruning, document)
|
|
721
|
+
pruneChanged = pruned !== art.content
|
|
722
|
+
candidates.push({ kind: 'art', html: pruned, title: art.title, legacy: gatePass })
|
|
723
|
+
if (pruneChanged) candidates.push({ kind: 'artu', html: art.content, title: art.title, legacy: false })
|
|
724
|
+
}
|
|
725
|
+
// `linkDensityModifier` is a public Readability option missing from the
|
|
726
|
+
// bundled type declarations; it relaxes the link-density cleaning cutoffs.
|
|
727
|
+
lenient = runReadability(document, { linkDensityModifier: 0.4 } as ConstructorParameters<typeof Readability>[1])
|
|
728
|
+
if (lenient !== null && lenient.content !== art?.content) {
|
|
729
|
+
candidates.push({ kind: 'artx', html: lenient.content, title: lenient.title, legacy: false })
|
|
730
|
+
}
|
|
599
731
|
}
|
|
600
|
-
const title = art?.title ?? lenient?.title
|
|
732
|
+
const title = art?.title ?? lenient?.title ?? (skipReadability ? fallbackTitle(document) : undefined)
|
|
601
733
|
|
|
602
734
|
const main = body === null ? null : findMainContainer(document)
|
|
603
|
-
if (main !== null) candidates.push({ kind: 'main', html: main.innerHTML, title, legacy: false })
|
|
735
|
+
if (main !== null) candidates.push({ kind: 'main', html: main.innerHTML, node: main, title, legacy: false })
|
|
604
736
|
|
|
605
|
-
candidates.push({ kind: 'doc', html: bodyHtml, legacy: !gatePass })
|
|
737
|
+
candidates.push({ kind: 'doc', html: bodyHtml, node: body ?? undefined, legacy: !gatePass })
|
|
606
738
|
|
|
607
739
|
if (body !== null) {
|
|
608
740
|
const cleaned = body.cloneNode(true) as Element
|
|
609
741
|
stripPageChrome(cleaned)
|
|
610
|
-
candidates.push({ kind: 'docc', html: cleaned.innerHTML, legacy: false })
|
|
742
|
+
candidates.push({ kind: 'docc', html: cleaned.innerHTML, node: cleaned, legacy: false })
|
|
611
743
|
}
|
|
612
744
|
|
|
613
745
|
const record = body === null ? null : findRecordRegion(body)
|
|
@@ -623,7 +755,7 @@ export function generateCandidates(document: Document): { candidates: RawCandida
|
|
|
623
755
|
unwrapNoscripts(unwrapped)
|
|
624
756
|
const nsArt = runReadability(unwrapped)
|
|
625
757
|
if (nsArt !== null) candidates.push({ kind: 'ns_art', html: prunedArticleContent(nsArt.content, unwrapped), title: nsArt.title, legacy: false })
|
|
626
|
-
candidates.push({ kind: 'ns_doc', html: unwrapped.body?.innerHTML ?? '', legacy: false })
|
|
758
|
+
candidates.push({ kind: 'ns_doc', html: unwrapped.body?.innerHTML ?? '', node: unwrapped.body ?? undefined, legacy: false })
|
|
627
759
|
}
|
|
628
760
|
|
|
629
761
|
const preloaded = discoursePreloadedHtml(document)
|
|
@@ -650,6 +782,7 @@ export function generateCandidates(document: Document): { candidates: RawCandida
|
|
|
650
782
|
hasMain: main !== null,
|
|
651
783
|
hasPreloaded: preloaded !== null,
|
|
652
784
|
main,
|
|
785
|
+
...(skipReadability ? { readabilitySkipped: true } : {}),
|
|
653
786
|
},
|
|
654
787
|
}
|
|
655
788
|
}
|
package/src/challenge.ts
CHANGED
|
@@ -43,8 +43,8 @@ export const CHALLENGE_POLL_INTERVAL_MS = 500
|
|
|
43
43
|
|
|
44
44
|
/**
|
|
45
45
|
* Tail of the fetch budget held back from the challenge wait so a cleared
|
|
46
|
-
* challenge still has time for its settle (
|
|
47
|
-
* content read and the denoise pipeline.
|
|
46
|
+
* challenge still has time for its settle (the `DEFAULT_SETTLE_MS` cap)
|
|
47
|
+
* plus the content read and the denoise pipeline.
|
|
48
48
|
*/
|
|
49
49
|
export const CHALLENGE_FINISH_RESERVE_MS = 8_000
|
|
50
50
|
|
package/src/config.ts
CHANGED
|
@@ -28,6 +28,15 @@ import z from '@deepseek-ai/schemastery'
|
|
|
28
28
|
*/
|
|
29
29
|
export const DEFAULT_SCROLL_BUDGET_MS = 20_000
|
|
30
30
|
|
|
31
|
+
/**
|
|
32
|
+
* Default cap (ms) on the post-navigation settle — the wait for SPA
|
|
33
|
+
* rendering to finish once the document has loaded (see the settle in
|
|
34
|
+
* `src/provider.ts`). A CAP, not a fixed cost: the settle ends as soon as
|
|
35
|
+
* the page's network goes quiet. Code-level only (tests, demo scripts):
|
|
36
|
+
* not part of the settings schema. `0` disables the settle entirely.
|
|
37
|
+
*/
|
|
38
|
+
export const DEFAULT_SETTLE_MS = 5_000
|
|
39
|
+
|
|
31
40
|
/** Default CDP endpoint when the settings section leaves it blank. */
|
|
32
41
|
export const DEFAULT_CDP_ENDPOINT = '127.0.0.1:9222'
|
|
33
42
|
|
|
@@ -149,6 +158,14 @@ export interface Config {
|
|
|
149
158
|
* configure it. Default: {@link DEFAULT_SCROLL_BUDGET_MS}.
|
|
150
159
|
*/
|
|
151
160
|
scrollBudgetMs?: number
|
|
161
|
+
/**
|
|
162
|
+
* Cap (ms) on the post-navigation settle (load baseline + bounded
|
|
163
|
+
* network-quiet window with stale-request discount). Code-level only
|
|
164
|
+
* (tests, demo scripts): not part of the settings schema, so users cannot
|
|
165
|
+
* configure it. `0` disables the settle. Default:
|
|
166
|
+
* {@link DEFAULT_SETTLE_MS}.
|
|
167
|
+
*/
|
|
168
|
+
settleMs?: number
|
|
152
169
|
/**
|
|
153
170
|
* How many fetches may render at once. Blank = backend default (4 for
|
|
154
171
|
* local — each slot launches a browser; 50 for CDP — each slot is a tab in
|
|
@@ -242,7 +259,7 @@ export function snapshotsOf(config: PlaywrightPluginConfig): Config {
|
|
|
242
259
|
export function resolveConfig(config: Partial<Config>): ResolvedConfig {
|
|
243
260
|
// The code-level budgets bypass the schema: they pass through untouched
|
|
244
261
|
// (only when explicitly given) and the effective* helpers default them.
|
|
245
|
-
const { challengeWaitMs, scrollBudgetMs, ...settings } = config
|
|
262
|
+
const { challengeWaitMs, scrollBudgetMs, settleMs, ...settings } = config
|
|
246
263
|
const resolved = Config(settings as never) as PlaywrightPluginConfig
|
|
247
264
|
return {
|
|
248
265
|
backend: resolved.backend.get(),
|
|
@@ -256,18 +273,19 @@ export function resolveConfig(config: Partial<Config>): ResolvedConfig {
|
|
|
256
273
|
challengeHttpFallback: resolved.challengeHttpFallback.get(),
|
|
257
274
|
...(challengeWaitMs === undefined ? {} : { challengeWaitMs }),
|
|
258
275
|
...(scrollBudgetMs === undefined ? {} : { scrollBudgetMs }),
|
|
276
|
+
...(settleMs === undefined ? {} : { settleMs }),
|
|
259
277
|
}
|
|
260
278
|
}
|
|
261
279
|
|
|
262
280
|
/**
|
|
263
281
|
* Complete config after schemastery applies the field defaults it owns.
|
|
264
282
|
* `maxConcurrency` stays optional because its effective default depends on
|
|
265
|
-
* `backend`; `challengeWaitMs` and `
|
|
266
|
-
* they are code-level overrides outside the schema —
|
|
267
|
-
* effective* helpers supply the built-in defaults.
|
|
283
|
+
* `backend`; `challengeWaitMs`, `scrollBudgetMs` and `settleMs` stay
|
|
284
|
+
* optional because they are code-level overrides outside the schema —
|
|
285
|
+
* absent, the effective* helpers supply the built-in defaults.
|
|
268
286
|
*/
|
|
269
|
-
export type ResolvedConfig = Omit<Required<Config>, 'maxConcurrency' | 'challengeWaitMs' | 'scrollBudgetMs'>
|
|
270
|
-
& { maxConcurrency?: number; challengeWaitMs?: number; scrollBudgetMs?: number }
|
|
287
|
+
export type ResolvedConfig = Omit<Required<Config>, 'maxConcurrency' | 'challengeWaitMs' | 'scrollBudgetMs' | 'settleMs'>
|
|
288
|
+
& { maxConcurrency?: number; challengeWaitMs?: number; scrollBudgetMs?: number; settleMs?: number }
|
|
271
289
|
|
|
272
290
|
/**
|
|
273
291
|
* The concurrency limit a fetch actually runs with: an explicit setting
|
|
@@ -329,6 +347,19 @@ export function effectiveScrollBudgetMs(config: Pick<Config, 'scrollBudgetMs'>):
|
|
|
329
347
|
return DEFAULT_SCROLL_BUDGET_MS
|
|
330
348
|
}
|
|
331
349
|
|
|
350
|
+
/**
|
|
351
|
+
* The settle cap a fetch actually runs with: an explicit value wins, else
|
|
352
|
+
* the built-in default. `0` means the settle is off (domcontentloaded
|
|
353
|
+
* content is read as-is).
|
|
354
|
+
*
|
|
355
|
+
* @param config - the resolved settings section (or any partial of it).
|
|
356
|
+
* @returns the bounded settle cap in milliseconds.
|
|
357
|
+
*/
|
|
358
|
+
export function effectiveSettleMs(config: Pick<Config, 'settleMs'>): number {
|
|
359
|
+
if (typeof config.settleMs === 'number') return config.settleMs
|
|
360
|
+
return DEFAULT_SETTLE_MS
|
|
361
|
+
}
|
|
362
|
+
|
|
332
363
|
/**
|
|
333
364
|
* The context mode a fetch actually runs with: profile (a tab in the remote
|
|
334
365
|
* browser's real profile) requires the CDP backend — a local launch has no
|