dsh-web-fetch-playwright 0.2.22 → 0.2.23

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -38,6 +38,15 @@ export interface RawCandidate {
38
38
  kind: CandidateKind;
39
39
  /** HTML of the extraction; sanitized by the caller. */
40
40
  html: string;
41
+ /**
42
+ * The extraction as a live DOM node, when the generator already holds one
43
+ * (`doc`/`docc`/`ns_doc` serialize `body`, `main` the container itself).
44
+ * DOMPurify sanitizes a node by importing a copy — the caller's re-parse
45
+ * of `html` (hundreds of ms on large pages, measured on the Angular docs
46
+ * badcase: 555ms for the whole-body candidate alone) is skipped; `html`
47
+ * stays the identity for the dedup below.
48
+ */
49
+ node?: Element;
41
50
  /** Title to prepend when the body does not open with a heading. */
42
51
  title?: string;
43
52
  /** Whether the 0.2.18 pipeline would have returned exactly this candidate. */
@@ -153,7 +162,46 @@ export interface GenerationFacts {
153
162
  hasPreloaded: boolean;
154
163
  /** The semantic main container, when found. */
155
164
  main: Element | null;
165
+ /**
166
+ * True when the Readability family was SKIPPED because nearly all body
167
+ * text is custom-wrapped ({@link customWrappedTextShare}) — every
168
+ * extraction that would carry it sanitizes to nothing. The orchestrator
169
+ * then seeds the selector with the empty-`art` reference stats such a run
170
+ * would have measured, keeping the surviving candidates' features
171
+ * (`words_vs_art_log`, the block diffs) exactly what a dead Readability
172
+ * pick yields.
173
+ */
174
+ readabilitySkipped?: boolean;
156
175
  }
176
+ /**
177
+ * At or above this share of custom-wrapped body text the Readability family
178
+ * is skipped (see {@link customWrappedTextShare}).
179
+ */
180
+ export declare const CUSTOM_WRAP_SKIP_SHARE = 0.9;
181
+ /**
182
+ * Share of the body's visible text that sits inside custom-element subtrees
183
+ * (tag names containing `-`: `<ms-app>`, `<my-doc>` — the HTML spec reserves
184
+ * them for custom elements; no standard HTML or MathML tag and no tag the
185
+ * denoise pipeline's DOMPurify step keeps contains a dash). DOMPurify's
186
+ * allowlist removes such tags wholesale — `KEEP_CONTENT: false` takes their
187
+ * subtrees with them — so once nearly ALL the page's text is custom-wrapped,
188
+ * every extraction that would carry it sanitizes to a corpse. Measured on
189
+ * the badcase (aistudio.google.com/docs/computer-use, an Angular app whose
190
+ * entire DOM lives under `<ms-app>`): 99% of body text custom-wrapped;
191
+ * Readability's pick sanitized to a root with ZERO words, the whole-document
192
+ * candidate to 39 of ~4.5k words — while the three Readability passes over
193
+ * the 1.1MB document cost ~6.5s of pure latency on candidates that could
194
+ * never win. The DOM-backed fallbacks, the noscript paths and the block
195
+ * classifier — the candidates that survive DOMPurify on this shape, and the
196
+ * ones the selector picks on such pages anyway — carry the extraction.
197
+ *
198
+ * Noise subtrees (script/style/nav/…) count as neither wrapped nor unwrapped;
199
+ * the share is over CONTENT-bearing text only. A partial share (a web
200
+ * component widget inside a normal article) stays below the threshold and
201
+ * keeps the full pipeline: only the shape where the family is provably dead
202
+ * pays for it.
203
+ */
204
+ export declare function customWrappedTextShare(body: Element): number;
157
205
  /**
158
206
  * Propose every candidate extraction for one parsed page.
159
207
  * @param document - the parsed page (data URIs already elided); not mutated.
@@ -41,8 +41,8 @@
41
41
  export declare const CHALLENGE_POLL_INTERVAL_MS = 500;
42
42
  /**
43
43
  * Tail of the fetch budget held back from the challenge wait so a cleared
44
- * challenge still has time for its settle (5s networkidle cap) plus the
45
- * content read and the denoise pipeline.
44
+ * challenge still has time for its settle (the `DEFAULT_SETTLE_MS` cap)
45
+ * plus the content read and the denoise pipeline.
46
46
  */
47
47
  export declare const CHALLENGE_FINISH_RESERVE_MS = 8000;
48
48
  /**
@@ -25,6 +25,14 @@ import z from '@deepseek-ai/schemastery';
25
25
  * skipped outright.
26
26
  */
27
27
  export declare const DEFAULT_SCROLL_BUDGET_MS = 20000;
28
+ /**
29
+ * Default cap (ms) on the post-navigation settle — the wait for SPA
30
+ * rendering to finish once the document has loaded (see the settle in
31
+ * `src/provider.ts`). A CAP, not a fixed cost: the settle ends as soon as
32
+ * the page's network goes quiet. Code-level only (tests, demo scripts):
33
+ * not part of the settings schema. `0` disables the settle entirely.
34
+ */
35
+ export declare const DEFAULT_SETTLE_MS = 5000;
28
36
  /** Default CDP endpoint when the settings section leaves it blank. */
29
37
  export declare const DEFAULT_CDP_ENDPOINT = "127.0.0.1:9222";
30
38
  /**
@@ -135,6 +143,14 @@ export interface Config {
135
143
  * configure it. Default: {@link DEFAULT_SCROLL_BUDGET_MS}.
136
144
  */
137
145
  scrollBudgetMs?: number;
146
+ /**
147
+ * Cap (ms) on the post-navigation settle (load baseline + bounded
148
+ * network-quiet window with stale-request discount). Code-level only
149
+ * (tests, demo scripts): not part of the settings schema, so users cannot
150
+ * configure it. `0` disables the settle. Default:
151
+ * {@link DEFAULT_SETTLE_MS}.
152
+ */
153
+ settleMs?: number;
138
154
  /**
139
155
  * How many fetches may render at once. Blank = backend default (4 for
140
156
  * local — each slot launches a browser; 50 for CDP — each slot is a tab in
@@ -214,14 +230,15 @@ export declare function resolveConfig(config: Partial<Config>): ResolvedConfig;
214
230
  /**
215
231
  * Complete config after schemastery applies the field defaults it owns.
216
232
  * `maxConcurrency` stays optional because its effective default depends on
217
- * `backend`; `challengeWaitMs` and `scrollBudgetMs` stay optional because
218
- * they are code-level overrides outside the schema — absent, the
219
- * effective* helpers supply the built-in defaults.
233
+ * `backend`; `challengeWaitMs`, `scrollBudgetMs` and `settleMs` stay
234
+ * optional because they are code-level overrides outside the schema —
235
+ * absent, the effective* helpers supply the built-in defaults.
220
236
  */
221
- export type ResolvedConfig = Omit<Required<Config>, 'maxConcurrency' | 'challengeWaitMs' | 'scrollBudgetMs'> & {
237
+ export type ResolvedConfig = Omit<Required<Config>, 'maxConcurrency' | 'challengeWaitMs' | 'scrollBudgetMs' | 'settleMs'> & {
222
238
  maxConcurrency?: number;
223
239
  challengeWaitMs?: number;
224
240
  scrollBudgetMs?: number;
241
+ settleMs?: number;
225
242
  };
226
243
  /**
227
244
  * The concurrency limit a fetch actually runs with: an explicit setting
@@ -264,6 +281,15 @@ export declare function effectiveChallengeHttpFallback(config: Pick<Config, 'cha
264
281
  * @returns the bounded walk budget in milliseconds.
265
282
  */
266
283
  export declare function effectiveScrollBudgetMs(config: Pick<Config, 'scrollBudgetMs'>): number;
284
+ /**
285
+ * The settle cap a fetch actually runs with: an explicit value wins, else
286
+ * the built-in default. `0` means the settle is off (domcontentloaded
287
+ * content is read as-is).
288
+ *
289
+ * @param config - the resolved settings section (or any partial of it).
290
+ * @returns the bounded settle cap in milliseconds.
291
+ */
292
+ export declare function effectiveSettleMs(config: Pick<Config, 'settleMs'>): number;
267
293
  /**
268
294
  * The context mode a fetch actually runs with: profile (a tab in the remote
269
295
  * browser's real profile) requires the CDP backend — a local launch has no
@@ -15,7 +15,7 @@
15
15
  */
16
16
  import type { Context } from '@deepseek-ai/cordis';
17
17
  import type { PlaywrightPluginConfig } from './config.ts';
18
- export { Config, DEFAULT_CDP_ENDPOINT, DEFAULT_CHALLENGE_HTTP_FALLBACK, DEFAULT_CHALLENGE_RETRIES, DEFAULT_CHALLENGE_WAIT_MS, DEFAULT_SCROLL_BUDGET_MS, effectiveChallengeHttpFallback, effectiveChallengeRetries, effectiveChallengeWaitMs, effectiveContextMode, effectiveScrollBudgetMs, normalizeCdpEndpoint, resolveConfig, snapshotsOf, } from './config.ts';
18
+ export { Config, DEFAULT_CDP_ENDPOINT, DEFAULT_CHALLENGE_HTTP_FALLBACK, DEFAULT_CHALLENGE_RETRIES, DEFAULT_CHALLENGE_WAIT_MS, DEFAULT_SCROLL_BUDGET_MS, DEFAULT_SETTLE_MS, effectiveChallengeHttpFallback, effectiveChallengeRetries, effectiveChallengeWaitMs, effectiveContextMode, effectiveScrollBudgetMs, effectiveSettleMs, normalizeCdpEndpoint, resolveConfig, snapshotsOf, } from './config.ts';
19
19
  export type { Config as PlaywrightFetchConfig, CdpContextMode, ContextIsolationScope, PlaywrightBackend, PlaywrightPluginConfig, ResolvedConfig, } from './config.ts';
20
20
  export { CdpConnectionPool } from './cdp-pool.ts';
21
21
  export type { CdpAcquireMode, CdpConnect, CdpLease } from './cdp-pool.ts';
@@ -18,6 +18,21 @@
18
18
  * refetch through the page's API request context
19
19
  * ({@link recoverDownloadResponse}) — instead of a `WEB_PROVIDER_ERROR`.
20
20
  *
21
+ * Transient transport failures (issue: `net::ERR_NETWORK_CHANGED` and its
22
+ * family) are retried, not terminal: Chromium's NetworkChangeNotifier kills
23
+ * every in-flight request with `ERR_NETWORK_CHANGED` the moment the HOST's
24
+ * network configuration shifts under the browser — a route/interface/DNS/
25
+ * proxy change, a VPN or DHCP flap, a Docker embedded-DNS rewrite; the page
26
+ * and its target are perfectly fine, only that one navigation died, and the
27
+ * very next `goto` usually succeeds (measured live: source.android.com
28
+ * failed once mid-navigation in one DSH session and fetched clean from the
29
+ * same host seconds later). Each navigation therefore retries the
30
+ * transport-blip family ({@link isTransientNavigationFailure} — deliberately
31
+ * NOT `ERR_ABORTED`, which is overloaded with the deterministic shapes the
32
+ * recoveries above already own) a bounded number of times with a short
33
+ * backoff, inside the same page and the remaining waiting budget; a blip
34
+ * that survives the attempt cap still fails as before.
35
+ *
21
36
  * Lifecycle: the local backend launches a browser per fetch and closes it —
22
37
  * nothing outlives the call. CDP session isolation keeps one connection per
23
38
  * DSH session until its context becomes idle; this also isolates gateways that
@@ -170,6 +185,12 @@ export declare class PlaywrightFetchProvider implements WebFetchProvider {
170
185
  protected readonly fetchTimeoutMs: number;
171
186
  /** Extraction grace after the waiting budget (ms); code-level, overridable by subclasses. */
172
187
  protected readonly extractGraceMs: number;
188
+ /** Settle quiet window (ms); code-level, overridable by subclasses (tests). */
189
+ protected readonly settleQuietMs: number;
190
+ /** Request age (ms) past which it stops gating the settle; overridable by subclasses (tests). */
191
+ protected readonly settleStaleMs: number;
192
+ /** Settle re-check cadence (ms); overridable by subclasses (tests). */
193
+ protected readonly settlePollMs: number;
173
194
  /** Shared CDP connection; injectable so the suite can substitute a fake. */
174
195
  protected readonly cdpPool: CdpConnectionPool;
175
196
  /**
@@ -219,6 +240,11 @@ export declare class PlaywrightFetchProvider implements WebFetchProvider {
219
240
  * status-only 204/205 — {@link recoverUnrenderedStatus}), and the response
220
241
  * being a download the browser refuses to commit as a document
221
242
  * (`Content-Disposition: attachment` — {@link recoverDownloadResponse}).
243
+ * A TRANSIENT transport failure (`net::ERR_NETWORK_CHANGED` and family)
244
+ * is neither: it carries no response and no render decision, only a
245
+ * momentary host-network blip that killed this one navigation — so it is
246
+ * retried on the SAME page, bounded by {@link MAX_TRANSIENT_NAV_RETRIES}
247
+ * attempts, a short backoff, and the remaining waiting budget.
222
248
  *
223
249
  * @returns the response `goto` resolved with, plus `recovered` when the
224
250
  * navigation failure was recovered into a finished result the caller must
@@ -244,5 +270,34 @@ export declare class PlaywrightFetchProvider implements WebFetchProvider {
244
270
  * @returns true when the challenge cleared; false when the budget ran out.
245
271
  */
246
272
  private waitForChallengeClear;
273
+ /**
274
+ * Best-effort settle for client-rendered content — a Lighthouse-style
275
+ * bounded network-quiet window replacing Playwright's `networkidle`
276
+ * (officially discouraged): worker-script fetches and cross-origin auth
277
+ * frames hold one request open on the page session FOREVER (their
278
+ * completion events are attributed to other CDP targets), so strict
279
+ * networkidle never arrives on exactly the modern pages the settle exists
280
+ * for, and the legacy wait burned its whole cap on them every fetch.
281
+ *
282
+ * Sequence: a `load` baseline (usually already past — domcontentloaded
283
+ * resolved earlier), then poll until no request has STARTED within
284
+ * {@link settleQuietMs} and every still-open request is older than
285
+ * {@link settleStaleMs} — a request that old is an attribution artifact
286
+ * or a long-poll whose completion no longer gates content. All waits
287
+ * share the {@link effectiveSettleMs} cap and the fetch deadline; a page
288
+ * that exposes no request events degrades to the legacy bounded
289
+ * networkidle wait. Never throws: any exit path keeps what the document
290
+ * already holds.
291
+ */
292
+ private settleForContent;
293
+ /**
294
+ * Settle outcome observation hook — silent in production by design (the
295
+ * settle is best-effort), but subclasses (tests, diagnostics) override it
296
+ * to see which exit path every fetch took: `off` (config), `quiet`
297
+ * (network went quiet), `stale-discounted` (only stale requests remained
298
+ * open), `cap` (window exhausted), `fallback` (no request events — the
299
+ * legacy networkidle wait ran), `aborted` (fetch deadline fired).
300
+ */
301
+ protected noteSettleOutcome(outcome: 'off' | 'quiet' | 'stale-discounted' | 'cap' | 'fallback' | 'aborted', elapsedMs: number): void;
247
302
  }
248
303
  export {};
@@ -56,6 +56,14 @@ export interface PlaywrightRequest {
56
56
  /** The frame that issued the request; compare with `page.mainFrame()`. */
57
57
  frame?(): unknown;
58
58
  }
59
+ /**
60
+ * A network request notification, as `page.on('request')` and its settled
61
+ * siblings deliver it. Identity only — the settle's network-quiet tracker
62
+ * keys its open-request map on the object itself.
63
+ */
64
+ export interface PlaywrightWebRequest {
65
+ url?(): string;
66
+ }
59
67
  /** A page inside a context. */
60
68
  export interface PlaywrightPage {
61
69
  goto(url: string, options?: {
@@ -81,6 +89,17 @@ export interface PlaywrightPage {
81
89
  * document). Absent on minimal fakes (content polling covers them).
82
90
  */
83
91
  on?(event: 'response', listener: (response: PlaywrightResponse) => void): unknown;
92
+ /**
93
+ * Request-start notification — the settle's network-quiet tracker. Absent
94
+ * on minimal fakes (the settle falls back to the legacy bounded
95
+ * networkidle wait).
96
+ */
97
+ on?(event: 'request', listener: (request: PlaywrightWebRequest) => void): unknown;
98
+ /**
99
+ * Request-settled notification (finished or failed) — closes the tracker's
100
+ * open-request bookkeeping. Absent on minimal fakes.
101
+ */
102
+ on?(event: 'requestfinished' | 'requestfailed', listener: (request: PlaywrightWebRequest) => void): unknown;
84
103
  /**
85
104
  * Download notification — a navigation the browser handed to its download
86
105
  * manager instead of committing a document (`Content-Disposition:
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "dsh-web-fetch-playwright",
3
- "version": "0.2.22",
3
+ "version": "0.2.23",
4
4
  "description": "Playwright/CDP web-fetch provider for DeepSeek Harness: renders pages in a real browser, denoises them (Readability + DOMPurify), and returns markdown.",
5
5
  "type": "module",
6
6
  "license": "MIT",
package/src/candidates.ts CHANGED
@@ -42,6 +42,15 @@ export interface RawCandidate {
42
42
  kind: CandidateKind
43
43
  /** HTML of the extraction; sanitized by the caller. */
44
44
  html: string
45
+ /**
46
+ * The extraction as a live DOM node, when the generator already holds one
47
+ * (`doc`/`docc`/`ns_doc` serialize `body`, `main` the container itself).
48
+ * DOMPurify sanitizes a node by importing a copy — the caller's re-parse
49
+ * of `html` (hundreds of ms on large pages, measured on the Angular docs
50
+ * badcase: 555ms for the whole-body candidate alone) is skipped; `html`
51
+ * stays the identity for the dedup below.
52
+ */
53
+ node?: Element
45
54
  /** Title to prepend when the body does not open with a heading. */
46
55
  title?: string
47
56
  /** Whether the 0.2.18 pipeline would have returned exactly this candidate. */
@@ -113,8 +122,20 @@ const GATE_MIN_ARTICLE_COVERAGE = 0.1
113
122
  export function articleCoversPage(articleHtml: string, document: Document): boolean {
114
123
  const bodyText = visibleTextLength(document.body)
115
124
  if (bodyText < GATE_MIN_BODY_TEXT_CHARS) return true
116
- const container = document.createElement('div')
117
- container.innerHTML = articleHtml
125
+ return articleCoversParsed(htmlFragment(articleHtml, document), bodyText)
126
+ }
127
+
128
+ /**
129
+ * {@link articleCoversPage} on an already-parsed article container, so the
130
+ * generator's gate and trailing prune share ONE parse of the article — the
131
+ * extraction is hundreds of KB on heavy pages (measured on the Angular docs
132
+ * badcase: ~400ms per parse of Readability's 485KB pick).
133
+ *
134
+ * @param container - the parsed article (read, never mutated).
135
+ * @param bodyText - the page body's visible text length, computed once.
136
+ */
137
+ function articleCoversParsed(container: Element, bodyText: number): boolean {
138
+ if (bodyText < GATE_MIN_BODY_TEXT_CHARS) return true
118
139
  const articleText = visibleTextLength(container)
119
140
  return articleText >= GATE_MIN_ARTICLE_TEXT_CHARS
120
141
  && articleText / bodyText >= GATE_MIN_ARTICLE_COVERAGE
@@ -190,8 +211,19 @@ function removeTrailingLinkDenseRun(parent: Element, articleText: number): void
190
211
 
191
212
  /** Prune with the 0.2.18 misfire guard; returns the original on a misfire. */
192
213
  export function prunedArticleContent(articleHtml: string, document: Document): string {
193
- const container = document.createElement('div')
194
- container.innerHTML = articleHtml
214
+ return pruneParsedArticle(htmlFragment(articleHtml, document), articleHtml)
215
+ }
216
+
217
+ /**
218
+ * {@link prunedArticleContent} on an already-parsed article container (the
219
+ * generator passes the gate's parse: the gate reads it first, then this
220
+ * mutates it — the same order the two-string-parses version produced).
221
+ *
222
+ * @param container - the parsed article; pruned in place.
223
+ * @param articleHtml - the string the container was parsed from (the
224
+ * misfire-guard's verbatim return).
225
+ */
226
+ function pruneParsedArticle(container: Element, articleHtml: string): string {
195
227
  const before = visibleTextLength(container)
196
228
  if (before === 0) return articleHtml
197
229
  pruneTrailingLinkDenseSections(container)
@@ -204,13 +236,25 @@ export function prunedArticleContent(articleHtml: string, document: Document): s
204
236
 
205
237
  // ------------------------------------------------- recommendation rails ----
206
238
 
207
- /** Link density at or above which a recommendation-named subtree is a rail. */
239
+ /**
240
+ * Link density at or above which a recommendation-named subtree is a rail.
241
+ */
208
242
  const RAIL_LINK_DENSITY = 0.3
209
243
  const RAIL_MIN_LINKS = 4
210
244
  /** Prose (low-link text) the page must keep outside the rails. */
211
245
  const RAIL_MIN_PROSE_CHARS = 80
212
246
  const RAIL_PROSE_LINK_DENSITY = 0.2
213
247
 
248
+ /**
249
+ * Necessary condition for {@link isRailName} on the raw class+id string: a
250
+ * matching token (`recommended-posts`, `Related`, …) always leaves one of
251
+ * these literals as a substring. Testing it first short-circuits the
252
+ * per-token split on the pages where it matters — {@link markRecommendationRails}
253
+ * walks every element, and on the measured 1.1MB Angular docs badcase the
254
+ * split alone cost ~360ms of the candidate generation.
255
+ */
256
+ const RAIL_NAME_HINT = /recommend|related|outbrain|taboola/
257
+
214
258
  /**
215
259
  * Class/id sub-tokens naming a recommendation / related-content rail: the
216
260
  * English words every CMS and theme uses, plus the two syndicated
@@ -225,6 +269,7 @@ function isRailName(token: string): boolean {
225
269
 
226
270
  function isRecommendationRail(element: Element): boolean {
227
271
  const names = `${element.getAttribute('class') ?? ''} ${element.id}`.toLowerCase()
272
+ if (!RAIL_NAME_HINT.test(names)) return false
228
273
  if (!names.split(/[\s_-]+/).some(isRailName)) return false
229
274
  if (element.querySelectorAll('a').length < RAIL_MIN_LINKS) return false
230
275
  const text = visibleTextLength(element)
@@ -562,11 +607,82 @@ export interface GenerationFacts {
562
607
  hasPreloaded: boolean
563
608
  /** The semantic main container, when found. */
564
609
  main: Element | null
610
+ /**
611
+ * True when the Readability family was SKIPPED because nearly all body
612
+ * text is custom-wrapped ({@link customWrappedTextShare}) — every
613
+ * extraction that would carry it sanitizes to nothing. The orchestrator
614
+ * then seeds the selector with the empty-`art` reference stats such a run
615
+ * would have measured, keeping the surviving candidates' features
616
+ * (`words_vs_art_log`, the block diffs) exactly what a dead Readability
617
+ * pick yields.
618
+ */
619
+ readabilitySkipped?: boolean
565
620
  }
566
621
 
567
622
  /** Below this much noscript text the noscript-unwrapped paths are skipped. */
568
623
  const NOSCRIPT_MIN_TEXT_CHARS = 200
569
624
 
625
+ /**
626
+ * At or above this share of custom-wrapped body text the Readability family
627
+ * is skipped (see {@link customWrappedTextShare}).
628
+ */
629
+ export const CUSTOM_WRAP_SKIP_SHARE = 0.9
630
+
631
+ /**
632
+ * Share of the body's visible text that sits inside custom-element subtrees
633
+ * (tag names containing `-`: `<ms-app>`, `<my-doc>` — the HTML spec reserves
634
+ * them for custom elements; no standard HTML or MathML tag and no tag the
635
+ * denoise pipeline's DOMPurify step keeps contains a dash). DOMPurify's
636
+ * allowlist removes such tags wholesale — `KEEP_CONTENT: false` takes their
637
+ * subtrees with them — so once nearly ALL the page's text is custom-wrapped,
638
+ * every extraction that would carry it sanitizes to a corpse. Measured on
639
+ * the badcase (aistudio.google.com/docs/computer-use, an Angular app whose
640
+ * entire DOM lives under `<ms-app>`): 99% of body text custom-wrapped;
641
+ * Readability's pick sanitized to a root with ZERO words, the whole-document
642
+ * candidate to 39 of ~4.5k words — while the three Readability passes over
643
+ * the 1.1MB document cost ~6.5s of pure latency on candidates that could
644
+ * never win. The DOM-backed fallbacks, the noscript paths and the block
645
+ * classifier — the candidates that survive DOMPurify on this shape, and the
646
+ * ones the selector picks on such pages anyway — carry the extraction.
647
+ *
648
+ * Noise subtrees (script/style/nav/…) count as neither wrapped nor unwrapped;
649
+ * the share is over CONTENT-bearing text only. A partial share (a web
650
+ * component widget inside a normal article) stays below the threshold and
651
+ * keeps the full pipeline: only the shape where the family is provably dead
652
+ * pays for it.
653
+ */
654
+ export function customWrappedTextShare(body: Element): number {
655
+ let total = 0
656
+ let wrapped = 0
657
+ const visit = (node: Node, insideCustom: boolean): void => {
658
+ if (node.nodeType === 3) {
659
+ const length = (node.nodeValue ?? '').replace(/\s+/g, '').length
660
+ if (length === 0) return
661
+ total += length
662
+ if (insideCustom) wrapped += length
663
+ return
664
+ }
665
+ if (node.nodeType !== 1 || NOISE_SUBTREE_TAGS.has(node.nodeName.toLowerCase())) return
666
+ const custom = insideCustom || node.nodeName.includes('-')
667
+ for (let child = node.firstChild; child !== null; child = child.nextSibling) visit(child, custom)
668
+ }
669
+ visit(body, false)
670
+ return total === 0 ? 0 : wrapped / total
671
+ }
672
+
673
+ /**
674
+ * The title the gated path hands its candidates: Readability's own first
675
+ * source is `og:title`, its second the `<title>` element — both remain
676
+ * readable on custom-element pages, where the DOM wrapping is the framework's
677
+ * but the metadata is the page's.
678
+ */
679
+ function fallbackTitle(document: Document): string | undefined {
680
+ const og = document.querySelector('meta[property="og:title"]')?.getAttribute('content')?.trim()
681
+ if (og !== undefined && og !== '') return og
682
+ const title = document.querySelector('title')?.textContent?.trim()
683
+ return title === undefined || title === '' ? undefined : title
684
+ }
685
+
570
686
  /**
571
687
  * Propose every candidate extraction for one parsed page.
572
688
  * @param document - the parsed page (data URIs already elided); not mutated.
@@ -576,38 +692,54 @@ export function generateCandidates(document: Document): { candidates: RawCandida
576
692
  const body = document.body
577
693
  const bodyHtml = body?.innerHTML ?? ''
578
694
 
579
- // Rails are marked before Readability (it drops the class names that
580
- // identify them) and cut from its output; the gate and `artu` see the
581
- // extraction exactly as unmarked, so the legacy marker and the "keep
582
- // everything" alternative are unchanged.
583
- const marked = runReadability(markRecommendationRails(document) ?? document)
584
- const rails = marked === null ? null : splitRails(marked.content, document)
585
- const art = marked === null || rails === null ? null : { ...marked, content: rails.plain }
586
- const gatePass = art !== null && articleCoversPage(art.content, document)
695
+ // The custom-element shape: skip the Readability family outright (see
696
+ // {@link customWrappedTextShare}) — its candidates sanitize to corpses.
697
+ const skipReadability = body !== null && customWrappedTextShare(body) >= CUSTOM_WRAP_SKIP_SHARE
698
+
699
+ let art: ReadabilityPick | null = null
700
+ let lenient: ReadabilityPick | null = null
701
+ let gatePass = false
587
702
  let pruneChanged = false
588
- if (art !== null) {
589
- const pruned = prunedArticleContent(rails?.derailed ?? art.content, document)
590
- pruneChanged = pruned !== art.content
591
- candidates.push({ kind: 'art', html: pruned, title: art.title, legacy: gatePass })
592
- if (pruneChanged) candidates.push({ kind: 'artu', html: art.content, title: art.title, legacy: false })
593
- }
594
- // `linkDensityModifier` is a public Readability option missing from the
595
- // bundled type declarations; it relaxes the link-density cleaning cutoffs.
596
- const lenient = runReadability(document, { linkDensityModifier: 0.4 } as ConstructorParameters<typeof Readability>[1])
597
- if (lenient !== null && lenient.content !== art?.content) {
598
- candidates.push({ kind: 'artx', html: lenient.content, title: lenient.title, legacy: false })
703
+ if (!skipReadability) {
704
+ // Rails are marked before Readability (it drops the class names that
705
+ // identify them) and cut from its output; the gate and `artu` see the
706
+ // extraction exactly as unmarked, so the legacy marker and the "keep
707
+ // everything" alternative are unchanged.
708
+ const marked = runReadability(markRecommendationRails(document) ?? document)
709
+ const rails = marked === null ? null : splitRails(marked.content, document)
710
+ art = marked === null || rails === null ? null : { ...marked, content: rails.plain }
711
+ if (art !== null) {
712
+ // One parse of the article serves the gate AND the prune (the common
713
+ // case: no rail was cut, so both read the same string).
714
+ const bodyText = visibleTextLength(body)
715
+ const parsed = htmlFragment(art.content, document)
716
+ gatePass = articleCoversParsed(parsed, bodyText)
717
+ const pruning = rails?.derailed ?? art.content
718
+ const pruned = pruning === art.content
719
+ ? pruneParsedArticle(parsed, art.content)
720
+ : prunedArticleContent(pruning, document)
721
+ pruneChanged = pruned !== art.content
722
+ candidates.push({ kind: 'art', html: pruned, title: art.title, legacy: gatePass })
723
+ if (pruneChanged) candidates.push({ kind: 'artu', html: art.content, title: art.title, legacy: false })
724
+ }
725
+ // `linkDensityModifier` is a public Readability option missing from the
726
+ // bundled type declarations; it relaxes the link-density cleaning cutoffs.
727
+ lenient = runReadability(document, { linkDensityModifier: 0.4 } as ConstructorParameters<typeof Readability>[1])
728
+ if (lenient !== null && lenient.content !== art?.content) {
729
+ candidates.push({ kind: 'artx', html: lenient.content, title: lenient.title, legacy: false })
730
+ }
599
731
  }
600
- const title = art?.title ?? lenient?.title
732
+ const title = art?.title ?? lenient?.title ?? (skipReadability ? fallbackTitle(document) : undefined)
601
733
 
602
734
  const main = body === null ? null : findMainContainer(document)
603
- if (main !== null) candidates.push({ kind: 'main', html: main.innerHTML, title, legacy: false })
735
+ if (main !== null) candidates.push({ kind: 'main', html: main.innerHTML, node: main, title, legacy: false })
604
736
 
605
- candidates.push({ kind: 'doc', html: bodyHtml, legacy: !gatePass })
737
+ candidates.push({ kind: 'doc', html: bodyHtml, node: body ?? undefined, legacy: !gatePass })
606
738
 
607
739
  if (body !== null) {
608
740
  const cleaned = body.cloneNode(true) as Element
609
741
  stripPageChrome(cleaned)
610
- candidates.push({ kind: 'docc', html: cleaned.innerHTML, legacy: false })
742
+ candidates.push({ kind: 'docc', html: cleaned.innerHTML, node: cleaned, legacy: false })
611
743
  }
612
744
 
613
745
  const record = body === null ? null : findRecordRegion(body)
@@ -623,7 +755,7 @@ export function generateCandidates(document: Document): { candidates: RawCandida
623
755
  unwrapNoscripts(unwrapped)
624
756
  const nsArt = runReadability(unwrapped)
625
757
  if (nsArt !== null) candidates.push({ kind: 'ns_art', html: prunedArticleContent(nsArt.content, unwrapped), title: nsArt.title, legacy: false })
626
- candidates.push({ kind: 'ns_doc', html: unwrapped.body?.innerHTML ?? '', legacy: false })
758
+ candidates.push({ kind: 'ns_doc', html: unwrapped.body?.innerHTML ?? '', node: unwrapped.body ?? undefined, legacy: false })
627
759
  }
628
760
 
629
761
  const preloaded = discoursePreloadedHtml(document)
@@ -650,6 +782,7 @@ export function generateCandidates(document: Document): { candidates: RawCandida
650
782
  hasMain: main !== null,
651
783
  hasPreloaded: preloaded !== null,
652
784
  main,
785
+ ...(skipReadability ? { readabilitySkipped: true } : {}),
653
786
  },
654
787
  }
655
788
  }
package/src/challenge.ts CHANGED
@@ -43,8 +43,8 @@ export const CHALLENGE_POLL_INTERVAL_MS = 500
43
43
 
44
44
  /**
45
45
  * Tail of the fetch budget held back from the challenge wait so a cleared
46
- * challenge still has time for its settle (5s networkidle cap) plus the
47
- * content read and the denoise pipeline.
46
+ * challenge still has time for its settle (the `DEFAULT_SETTLE_MS` cap)
47
+ * plus the content read and the denoise pipeline.
48
48
  */
49
49
  export const CHALLENGE_FINISH_RESERVE_MS = 8_000
50
50
 
package/src/config.ts CHANGED
@@ -28,6 +28,15 @@ import z from '@deepseek-ai/schemastery'
28
28
  */
29
29
  export const DEFAULT_SCROLL_BUDGET_MS = 20_000
30
30
 
31
+ /**
32
+ * Default cap (ms) on the post-navigation settle — the wait for SPA
33
+ * rendering to finish once the document has loaded (see the settle in
34
+ * `src/provider.ts`). A CAP, not a fixed cost: the settle ends as soon as
35
+ * the page's network goes quiet. Code-level only (tests, demo scripts):
36
+ * not part of the settings schema. `0` disables the settle entirely.
37
+ */
38
+ export const DEFAULT_SETTLE_MS = 5_000
39
+
31
40
  /** Default CDP endpoint when the settings section leaves it blank. */
32
41
  export const DEFAULT_CDP_ENDPOINT = '127.0.0.1:9222'
33
42
 
@@ -149,6 +158,14 @@ export interface Config {
149
158
  * configure it. Default: {@link DEFAULT_SCROLL_BUDGET_MS}.
150
159
  */
151
160
  scrollBudgetMs?: number
161
+ /**
162
+ * Cap (ms) on the post-navigation settle (load baseline + bounded
163
+ * network-quiet window with stale-request discount). Code-level only
164
+ * (tests, demo scripts): not part of the settings schema, so users cannot
165
+ * configure it. `0` disables the settle. Default:
166
+ * {@link DEFAULT_SETTLE_MS}.
167
+ */
168
+ settleMs?: number
152
169
  /**
153
170
  * How many fetches may render at once. Blank = backend default (4 for
154
171
  * local — each slot launches a browser; 50 for CDP — each slot is a tab in
@@ -242,7 +259,7 @@ export function snapshotsOf(config: PlaywrightPluginConfig): Config {
242
259
  export function resolveConfig(config: Partial<Config>): ResolvedConfig {
243
260
  // The code-level budgets bypass the schema: they pass through untouched
244
261
  // (only when explicitly given) and the effective* helpers default them.
245
- const { challengeWaitMs, scrollBudgetMs, ...settings } = config
262
+ const { challengeWaitMs, scrollBudgetMs, settleMs, ...settings } = config
246
263
  const resolved = Config(settings as never) as PlaywrightPluginConfig
247
264
  return {
248
265
  backend: resolved.backend.get(),
@@ -256,18 +273,19 @@ export function resolveConfig(config: Partial<Config>): ResolvedConfig {
256
273
  challengeHttpFallback: resolved.challengeHttpFallback.get(),
257
274
  ...(challengeWaitMs === undefined ? {} : { challengeWaitMs }),
258
275
  ...(scrollBudgetMs === undefined ? {} : { scrollBudgetMs }),
276
+ ...(settleMs === undefined ? {} : { settleMs }),
259
277
  }
260
278
  }
261
279
 
262
280
  /**
263
281
  * Complete config after schemastery applies the field defaults it owns.
264
282
  * `maxConcurrency` stays optional because its effective default depends on
265
- * `backend`; `challengeWaitMs` and `scrollBudgetMs` stay optional because
266
- * they are code-level overrides outside the schema — absent, the
267
- * effective* helpers supply the built-in defaults.
283
+ * `backend`; `challengeWaitMs`, `scrollBudgetMs` and `settleMs` stay
284
+ * optional because they are code-level overrides outside the schema —
285
+ * absent, the effective* helpers supply the built-in defaults.
268
286
  */
269
- export type ResolvedConfig = Omit<Required<Config>, 'maxConcurrency' | 'challengeWaitMs' | 'scrollBudgetMs'>
270
- & { maxConcurrency?: number; challengeWaitMs?: number; scrollBudgetMs?: number }
287
+ export type ResolvedConfig = Omit<Required<Config>, 'maxConcurrency' | 'challengeWaitMs' | 'scrollBudgetMs' | 'settleMs'>
288
+ & { maxConcurrency?: number; challengeWaitMs?: number; scrollBudgetMs?: number; settleMs?: number }
271
289
 
272
290
  /**
273
291
  * The concurrency limit a fetch actually runs with: an explicit setting
@@ -329,6 +347,19 @@ export function effectiveScrollBudgetMs(config: Pick<Config, 'scrollBudgetMs'>):
329
347
  return DEFAULT_SCROLL_BUDGET_MS
330
348
  }
331
349
 
350
+ /**
351
+ * The settle cap a fetch actually runs with: an explicit value wins, else
352
+ * the built-in default. `0` means the settle is off (domcontentloaded
353
+ * content is read as-is).
354
+ *
355
+ * @param config - the resolved settings section (or any partial of it).
356
+ * @returns the bounded settle cap in milliseconds.
357
+ */
358
+ export function effectiveSettleMs(config: Pick<Config, 'settleMs'>): number {
359
+ if (typeof config.settleMs === 'number') return config.settleMs
360
+ return DEFAULT_SETTLE_MS
361
+ }
362
+
332
363
  /**
333
364
  * The context mode a fetch actually runs with: profile (a tab in the remote
334
365
  * browser's real profile) requires the CDP backend — a local launch has no