dsh-web-fetch-playwright 0.2.20 → 0.2.23

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/provider.ts CHANGED
@@ -18,6 +18,21 @@
18
18
  * refetch through the page's API request context
19
19
  * ({@link recoverDownloadResponse}) — instead of a `WEB_PROVIDER_ERROR`.
20
20
  *
21
+ * Transient transport failures (issue: `net::ERR_NETWORK_CHANGED` and its
22
+ * family) are retried, not terminal: Chromium's NetworkChangeNotifier kills
23
+ * every in-flight request with `ERR_NETWORK_CHANGED` the moment the HOST's
24
+ * network configuration shifts under the browser — a route/interface/DNS/
25
+ * proxy change, a VPN or DHCP flap, a Docker embedded-DNS rewrite; the page
26
+ * and its target are perfectly fine, only that one navigation died, and the
27
+ * very next `goto` usually succeeds (measured live: source.android.com
28
+ * failed once mid-navigation in one DSH session and fetched clean from the
29
+ * same host seconds later). Each navigation therefore retries the
30
+ * transport-blip family ({@link isTransientNavigationFailure} — deliberately
31
+ * NOT `ERR_ABORTED`, which is overloaded with the deterministic shapes the
32
+ * recoveries above already own) a bounded number of times with a short
33
+ * backoff, inside the same page and the remaining waiting budget; a blip
34
+ * that survives the attempt cap still fails as before.
35
+ *
21
36
  * Lifecycle: the local backend launches a browser per fetch and closes it —
22
37
  * nothing outlives the call. CDP session isolation keeps one connection per
23
38
  * DSH session until its context becomes idle; this also isolates gateways that
@@ -55,9 +70,14 @@
55
70
  * behavior). The wait tracks the LAST main-frame navigation response (the
56
71
  * real page reloads in after the challenge clears) and watches the live DOM
57
72
  * so SPA-style clears are caught too. It never clicks, never injects
58
- * answers, never touches cookies itself; when the budget runs out the fetch
59
- * fails with {@link WEB_FETCH_CHALLENGE_CODE} instead of returning the
60
- * interstitial as content.
73
+ * answers, never touches cookies itself; when the budget runs out, ONE
74
+ * plain-HTTP refetch of the same URL through the page's request context
75
+ * (same cookie jar, the browser's UA, none of Chromium's render decisions
76
+ * — `challengeHttpFallback`, default on) tries to retrieve the real
77
+ * document, gated by the same response-level and content-level challenge
78
+ * classifiers so an interstitial can never pass as content. When that too
79
+ * fails, the fetch fails with {@link WEB_FETCH_CHALLENGE_CODE} instead of
80
+ * returning the interstitial as content.
61
81
  *
62
82
  * Deep capture (viewport-virtualized feeds and app-shell documents): feed-
63
83
  * shaped pages — Discourse topic streams, comment threads, timelines —
@@ -84,6 +104,17 @@
84
104
  * degrades to the legacy capture (see `src/autoscroll.ts` for the measured
85
105
  * linux.do and Feishu badcases).
86
106
  *
107
+ * Deadline (soft, then hard): each fetch has a {@link DEFAULT_TIMEOUT_MS}
108
+ * budget for WAITING — navigation, settle, challenge wait, scroll walk. When
109
+ * it runs out the page is NOT torn down: every wait is bounded by the same
110
+ * budget and simply ends, a navigation still loading is stopped
111
+ * (`window.stop()`) once its main-frame response has arrived, and whatever
112
+ * the page holds by then goes through the normal extraction — a slow or
113
+ * endless page still hands the agent the content it already rendered
114
+ * (flagged `truncated` when the document never finished loading). Only when
115
+ * nothing arrived at all, or extraction itself hangs past
116
+ * {@link EXTRACT_GRACE_MS}, does the fetch fail with `WEB_FETCH_TIMEOUT`.
117
+ *
87
118
  * @module dsh-web-fetch-playwright/provider
88
119
  */
89
120
 
@@ -91,15 +122,15 @@ import { WebError } from '@deepseek-ai/dsh-web'
91
122
  import type { WebFetchProvider, WebFetchRequest, WebFetchResult } from '@deepseek-ai/dsh-web'
92
123
  import { CHALLENGE_DOM_PROBE, CHALLENGE_FINISH_RESERVE_MS, CHALLENGE_POLL_INTERVAL_MS, classifyChallengeHtml, classifyChallengeResponse, isChallengeCompatibleResponse } from './challenge.ts'
93
124
  import type { ChallengeVerdict } from './challenge.ts'
94
- import { deepCaptureWalk, SCROLL_FINISH_RESERVE_MS } from './autoscroll.ts'
95
- import { DEFAULT_MAX_CONCURRENCY_CDP, DEFAULT_MAX_CONCURRENCY_LOCAL, effectiveChallengeRetries, effectiveChallengeWaitMs, effectiveContextMode, effectiveMaxConcurrency, effectiveScrollBudgetMs, normalizeCdpEndpoint } from './config.ts'
125
+ import { deepCaptureWalk } from './autoscroll.ts'
126
+ import { DEFAULT_MAX_CONCURRENCY_CDP, DEFAULT_MAX_CONCURRENCY_LOCAL, effectiveChallengeHttpFallback, effectiveChallengeRetries, effectiveChallengeWaitMs, effectiveContextMode, effectiveMaxConcurrency, effectiveScrollBudgetMs, effectiveSettleMs, normalizeCdpEndpoint } from './config.ts'
96
127
  import type { ResolvedConfig } from './config.ts'
97
128
  import { CdpConnectionPool } from './cdp-pool.ts'
98
129
  import { htmlToMarkdown } from './markdown.ts'
99
130
  import { extractPdfMarkdown, isPdfBytes, isPdfContentType, mayBePdf, MAX_PDF_BYTES, WEB_PDF_INVALID_CODE, WEB_PDF_TOO_LARGE_CODE } from './pdf.ts'
100
131
  import { STRIP_SCRIPTING_ELEMENTS } from './strip.ts'
101
132
  import { resolveCdpBackend, resolvePlaywrightBackend } from './playwright-resolve.ts'
102
- import type { PlaywrightBrowser, PlaywrightContext, PlaywrightDownload, PlaywrightPage, PlaywrightResponse, PlaywrightRoute } from './types.ts'
133
+ import type { PlaywrightBrowser, PlaywrightContext, PlaywrightDownload, PlaywrightPage, PlaywrightResponse, PlaywrightRoute, PlaywrightWebRequest } from './types.ts'
103
134
 
104
135
  /** Stable id this provider registers under (the bundle patch pins it). */
105
136
  export const PLAYWRIGHT_FETCH_PROVIDER_ID = 'playwright'
@@ -128,13 +159,50 @@ const MAX_PIPELINE_INPUT_CHARS = 2_000_000
128
159
  * budget to render after dequeue — and failing fast tells the caller to
129
160
  * retry or raise `maxConcurrency` instead of hanging until an abort.
130
161
  */
131
- const QUEUE_TIMEOUT_MS = 20_000
162
+ const QUEUE_TIMEOUT_MS = 10_000
163
+
164
+ /**
165
+ * Per-fetch waiting budget (ms): when it runs out the fetch stops waiting
166
+ * for the page and extracts what it already holds (see the module docs).
167
+ */
168
+ const DEFAULT_TIMEOUT_MS = 25_000
169
+
170
+ /**
171
+ * Hard backstop (ms) after {@link DEFAULT_TIMEOUT_MS} for reading and
172
+ * decoding what the page holds; only past it is the tab closed and the fetch
173
+ * failed. Budget plus grace stays inside the tool layer's 60s.
174
+ */
175
+ const EXTRACT_GRACE_MS = 10_000
176
+
177
+ /** Bound (ms) on the best-effort `window.stop()` of a still-loading page. */
178
+ const STOP_LOADING_GRACE_MS = 1_000
132
179
 
133
- /** Default per-fetch budget (ms), inside the tool layer's 60s. */
134
- const DEFAULT_TIMEOUT_MS = 45_000
180
+ /**
181
+ * Settle quiet window (ms): the post-load settle ends once no request has
182
+ * STARTED for this long AND every still-open request is stale (see
183
+ * {@link SETTLE_STALE_MS}) — the Lighthouse-style network-quiet threshold,
184
+ * sized to match Playwright's own networkidle quiet window.
185
+ */
186
+ const SETTLE_QUIET_MS = 500
187
+
188
+ /**
189
+ * A request open longer than this (ms) stops gating the settle. On the
190
+ * pages that motivated the custom detector, the long-lived request is a
191
+ * worker-script fetch or a cross-origin auth frame whose completion events
192
+ * are attributed to another CDP target and NEVER arrive on the page session
193
+ * (measured live over CDP: platform.claude.com's `turbopack-worker-*.js`
194
+ * and aistudio.google.com's `accounts.google.com/ServiceLogin` probe stayed
195
+ * "open" for the page's whole life while the page itself completed at
196
+ * `load`), so strict `networkidle` — Playwright's is officially discouraged
197
+ * — is unreachable there and the legacy wait burned its whole cap on every
198
+ * fetch. A genuine content request this old is the documented trade: the
199
+ * settle stops waiting for it, and the scroll walk plus the live read still
200
+ * capture whatever it renders when it lands.
201
+ */
202
+ const SETTLE_STALE_MS = 3_000
135
203
 
136
- /** Best-effort post-DOM settle wait (ms) so SPA content can finish rendering. */
137
- const SETTLE_MS = 5_000
204
+ /** How often the settle re-checks its quiet/stale conditions (ms). */
205
+ const SETTLE_POLL_MS = 100
138
206
 
139
207
  /**
140
208
  * Grace period (ms) for context/browser closes in the cleanup path. A wedged
@@ -143,6 +211,30 @@ const SETTLE_MS = 5_000
143
211
  */
144
212
  const CLOSE_GRACE_MS = 2_000
145
213
 
214
+ /**
215
+ * How many times a navigation may be re-issued after a TRANSIENT transport
216
+ * failure (`net::ERR_NETWORK_CHANGED` and family — see
217
+ * {@link isTransientNavigationFailure}) before the fetch gives up. Three
218
+ * total navigations cover the single-blip badcase and the rare double-blip
219
+ * while capping the worst-case latency a persistently flapping network can
220
+ * add.
221
+ */
222
+ const MAX_TRANSIENT_NAV_RETRIES = 2
223
+
224
+ /**
225
+ * Backoff (ms) before each transient-navigation retry, indexed by the retry
226
+ * number: short first (most blips clear in well under a second), longer
227
+ * second. Bounded by the fetch deadline via an abortable sleep.
228
+ */
229
+ const TRANSIENT_RETRY_BACKOFF_MS = [250, 750]
230
+
231
+ /**
232
+ * Waiting budget (ms) that must REMAIN before a transient-navigation retry
233
+ * may start: the retry must not eat the tail the settle/decode path still
234
+ * needs, and a re-navigation that cannot finish is pure waste.
235
+ */
236
+ const TRANSIENT_RETRY_RESERVE_MS = 2_000
237
+
146
238
  /**
147
239
  * One render session: a browser, the context this fetch works in, and the
148
240
  * tab it owns. Local backends own the browser too; CDP sessions ride the
@@ -163,9 +255,13 @@ export interface BrowserSession {
163
255
  }
164
256
 
165
257
  /**
166
- * A single abort deadline composing the caller's signal with this provider's
167
- * time budget. The timeout reason is kept apart from an outer abort so the
168
- * error translation can pick `WEB_FETCH_TIMEOUT` over `WEB_ABORTED`.
258
+ * The fetch's two-stage deadline composed with the caller's signal. The SOFT
259
+ * stage (`timeoutMs`) only bounds waiting: {@link remainingMs} counts down to
260
+ * it and {@link expired} reports it, so every wait ends there and the fetch
261
+ * extracts what the page holds. The HARD stage (`timeoutMs + graceMs`) aborts
262
+ * {@link signal}, which closes the page. The timeout reason is kept apart
263
+ * from an outer abort so the error translation can pick `WEB_FETCH_TIMEOUT`
264
+ * over `WEB_ABORTED`.
169
265
  */
170
266
  class Deadline {
171
267
  readonly signal: AbortSignal
@@ -173,13 +269,13 @@ class Deadline {
173
269
  private readonly expiresAt: number
174
270
  private timedOut = false
175
271
 
176
- constructor(outer: AbortSignal | undefined, timeoutMs: number) {
272
+ constructor(outer: AbortSignal | undefined, timeoutMs: number, graceMs = 0) {
177
273
  this.signal = this.controller.signal
178
274
  this.expiresAt = Date.now() + timeoutMs
179
275
  const timer = setTimeout(() => {
180
276
  this.timedOut = true
181
277
  this.controller.abort(new Error('playwright fetch deadline'))
182
- }, timeoutMs)
278
+ }, timeoutMs + graceMs)
183
279
  if (outer !== undefined) {
184
280
  if (outer.aborted) this.controller.abort(outer.reason)
185
281
  else outer.addEventListener('abort', () => { this.controller.abort(outer.reason) }, { once: true })
@@ -187,12 +283,17 @@ class Deadline {
187
283
  this.signal.addEventListener('abort', () => { clearTimeout(timer) }, { once: true })
188
284
  }
189
285
 
190
- /** Whether OUR timer fired (vs an outer cancellation). */
286
+ /** Whether OUR hard timer fired (vs an outer cancellation). */
191
287
  get isTimeout(): boolean {
192
288
  return this.timedOut
193
289
  }
194
290
 
195
- /** Milliseconds left on the budget, floored at 1 for Playwright options. */
291
+ /** Whether the soft (waiting) budget has run out. */
292
+ get expired(): boolean {
293
+ return Date.now() >= this.expiresAt
294
+ }
295
+
296
+ /** Milliseconds left on the waiting budget, floored at 1 for Playwright options. */
196
297
  remainingMs(): number {
197
298
  return Math.max(1, this.expiresAt - Date.now())
198
299
  }
@@ -324,6 +425,9 @@ function translateError(error: unknown, deadline: Deadline): WebError {
324
425
  return new WebError('web fetch aborted', 'WEB_ABORTED', { cause: error })
325
426
  }
326
427
  if (error instanceof WebError) return error
428
+ // Past the waiting budget with nothing salvageable (no response arrived,
429
+ // or a bounded wait's own timeout surfaced): the budget is the cause.
430
+ if (deadline.expired) return new WebError('playwright web fetch timed out before the page delivered any content', 'WEB_FETCH_TIMEOUT', { cause: error })
327
431
  return new WebError(`playwright web fetch failed: ${String(error instanceof Error ? error.message : error)}`, 'WEB_PROVIDER_ERROR', { cause: error })
328
432
  }
329
433
 
@@ -337,6 +441,21 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
337
441
 
338
442
  private readonly semaphore = new Semaphore(DEFAULT_MAX_CONCURRENCY_LOCAL)
339
443
 
444
+ /** Waiting budget per fetch (ms); code-level, overridable by subclasses (tests). */
445
+ protected readonly fetchTimeoutMs: number = DEFAULT_TIMEOUT_MS
446
+
447
+ /** Extraction grace after the waiting budget (ms); code-level, overridable by subclasses. */
448
+ protected readonly extractGraceMs: number = EXTRACT_GRACE_MS
449
+
450
+ /** Settle quiet window (ms); code-level, overridable by subclasses (tests). */
451
+ protected readonly settleQuietMs: number = SETTLE_QUIET_MS
452
+
453
+ /** Request age (ms) past which it stops gating the settle; overridable by subclasses (tests). */
454
+ protected readonly settleStaleMs: number = SETTLE_STALE_MS
455
+
456
+ /** Settle re-check cadence (ms); overridable by subclasses (tests). */
457
+ protected readonly settlePollMs: number = SETTLE_POLL_MS
458
+
340
459
  /** Shared CDP connection; injectable so the suite can substitute a fake. */
341
460
  protected readonly cdpPool: CdpConnectionPool
342
461
 
@@ -382,7 +501,7 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
382
501
  }
383
502
  }
384
503
  const url = validateFetchUrl(request.url)
385
- const deadline = new Deadline(signal, DEFAULT_TIMEOUT_MS)
504
+ const deadline = new Deadline(signal, this.fetchTimeoutMs, this.extractGraceMs)
386
505
 
387
506
  let session: BrowserSession | undefined
388
507
  let acquired = false
@@ -394,6 +513,7 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
394
513
  acquired = true
395
514
  session = await this.openSession(config, deadline, sessionId)
396
515
  if (deadline.signal.aborted) throw new Error('web fetch aborted during session setup')
516
+ if (deadline.expired) throw new Error('the fetch budget ran out during session setup')
397
517
  // An aborted deadline must also interrupt Playwright's own waits:
398
518
  // closing the page rejects every pending operation on it (and, for
399
519
  // fetch-owned contexts, the context close that follows takes the rest).
@@ -501,15 +621,22 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
501
621
  const page = session.page
502
622
  const challengeWaitMs = effectiveChallengeWaitMs(config)
503
623
  // The tracker is always on: the challenge wait needs the LAST main-frame
504
- // response (feature switch below only gates the waiting), and status
505
- // recovery needs whatever response arrived before a failed goto.
624
+ // response (feature switch below only gates the waiting), status
625
+ // recovery needs whatever response arrived before a failed goto, and
626
+ // the settle needs the page's open-request bookkeeping from BEFORE
627
+ // navigation — a request that started before the settle begins still
628
+ // counts toward quiet and staleness.
506
629
  const tracker = trackMainFrameResponses(page)
630
+ const activity = trackRequestActivity(page)
507
631
  // Feature switch: 0 keeps the exact legacy (pre-0.2.5) behavior — the
508
632
  // first response decides, no waiting — an escape hatch and the A/B
509
633
  // baseline every test proves the bug against.
510
634
  const first = await this.navigate(page, url, tracker, deadline, config.denoise)
511
635
  if (first.recovered !== undefined) return first.recovered
512
636
  let response = first.response
637
+ // The document never finished loading inside the budget: extract what
638
+ // arrived and flag the result as incomplete.
639
+ let partial = first.partial === true
513
640
  tracker.seed(response)
514
641
 
515
642
  let challengeEntryResponse: PlaywrightResponse | null = null
@@ -536,7 +663,18 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
536
663
  try { chained = classifyChallengeHtml(await page.content()) === 'challenge' } catch { chained = false }
537
664
  if (!chained) break
538
665
  }
539
- if (--attemptsLeft <= 0) {
666
+ // No budget left for another window: a re-navigation could only
667
+ // land on the interstitial again.
668
+ if (--attemptsLeft <= 0 || deadline.expired) {
669
+ // The dead-end refetch: many Cloudflare edges serve the real
670
+ // document to plain-HTTP clients (same cookies, no render
671
+ // decision) while withholding the interactive challenge from
672
+ // THIS browser session — measured live on v2ex.com, where the
673
+ // browser drew an empty widget frame and curl drew the article.
674
+ if (effectiveChallengeHttpFallback(config)) {
675
+ const fallback = await this.challengeHttpFallback(page, url, deadline, config.denoise)
676
+ if (fallback !== undefined) return fallback
677
+ }
540
678
  const lastStatus = (tracker.last() ?? response)?.status()
541
679
  throw new WebError(
542
680
  `the site kept serving a Cloudflare challenge (last status ${lastStatus === undefined ? 'unknown' : String(lastStatus)}) for up to ${String(challengeWaitMs)}ms across ${String(effectiveChallengeRetries(config) + 1)} attempt(s); the browser did not clear it naturally — retry later, raise challengeWaitMs, or use a profile whose browser already holds clearance`,
@@ -549,6 +687,7 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
549
687
  const retry = await this.navigate(page, url, tracker, deadline, config.denoise)
550
688
  if (retry.recovered !== undefined) return retry.recovered
551
689
  response = retry.response
690
+ partial = retry.partial === true
552
691
  tracker.seed(response)
553
692
  }
554
693
  }
@@ -577,24 +716,29 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
577
716
 
578
717
  // Non-HTML decodes straight from the response body; no denoise applies.
579
718
  if (kind === 'text') {
580
- const text = finalResponse !== null ? await finalResponse.text() : await page.content()
581
- return capResult(finalUrl, statusCode, { kind: 'text', content: text })
719
+ // A stopped load has no complete response body to read; the page's
720
+ // rendered text is what arrived.
721
+ const text = partial
722
+ ? await renderedText(page)
723
+ : finalResponse !== null ? await finalResponse.text() : await page.content()
724
+ return flagPartial(capResult(finalUrl, statusCode, { kind: 'text', content: text }), partial)
582
725
  }
583
726
 
584
727
  // Best-effort settle for client-rendered content; a timeout just keeps
585
- // what domcontentloaded already produced.
586
- await page.waitForLoadState('networkidle', { timeout: Math.min(SETTLE_MS, deadline.remainingMs()) }).catch(() => {})
728
+ // what domcontentloaded already produced. See {@link settleForContent}
729
+ // for why this is a custom quiet window rather than Playwright's
730
+ // networkidle.
731
+ await this.settleForContent(page, deadline, config, activity)
587
732
 
588
733
  // Deep capture (viewport-virtualized feeds and app-shell documents lose
589
734
  // their off-window content at rest — see `src/autoscroll.ts`): a bounded
590
735
  // walk stashes what each rendered window shows and, when it restored
591
736
  // anything, hands back its own merged, already-stripped serialization.
592
- // HTML documents only, inside the deadline with a decode reserve,
593
- // silently degrading to the legacy capture; `scrollBudgetMs: 0` skips it.
594
- const scrollBudgetMs = Math.min(
595
- effectiveScrollBudgetMs(config),
596
- deadline.remainingMs() - SCROLL_FINISH_RESERVE_MS,
597
- )
737
+ // HTML documents only, ending at the waiting budget at the latest — the
738
+ // walk then hands back what it stashed so far and extraction runs in
739
+ // the grace window — silently degrading to the legacy capture;
740
+ // `scrollBudgetMs: 0` skips it, as does a budget already spent.
741
+ const scrollBudgetMs = deadline.expired ? 0 : Math.min(effectiveScrollBudgetMs(config), deadline.remainingMs())
598
742
  const captured = scrollBudgetMs > 0 ? await deepCaptureWalk(page, scrollBudgetMs) : undefined
599
743
 
600
744
  // Serialization safety for the live read: the document is about to be
@@ -613,14 +757,56 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
613
757
  if (!config.denoise) {
614
758
  // The tool layer's own turndown renders raw HTML; the checkbox only
615
759
  // governs the Readability/DOMPurify stage this provider owns.
616
- return capResult(finalUrl, statusCode, { kind: 'html', content: html })
760
+ return flagPartial(capResult(finalUrl, statusCode, { kind: 'html', content: html }), partial)
761
+ }
762
+ return flagPartial(denoiseResult(finalUrl, statusCode, html), partial)
763
+ }
764
+
765
+ /**
766
+ * The challenge dead-end's plain-HTTP refetch: one request through the
767
+ * page's own request context (the SAME cookie jar — any clearance the
768
+ * profile holds rides along — but none of Chromium's render decisions),
769
+ * wearing the browser's own User-Agent. Returns the finished result when
770
+ * the answer is a NON-challenge HTML document, undefined otherwise (the
771
+ * caller throws the honest WEB_FETCH_CHALLENGE). A challenge served to
772
+ * this request too, a non-HTML body, or any transport failure keeps the
773
+ * browser-only verdict.
774
+ */
775
+ private async challengeHttpFallback(page: PlaywrightPage, url: URL, deadline: Deadline, denoise: boolean): Promise<WebFetchResult | undefined> {
776
+ if (page.request === undefined || deadline.expired || deadline.signal.aborted) return undefined
777
+ try {
778
+ // A CDP-attached browser's request context does not inherit the
779
+ // browser's UA — send it explicitly so the edge sees one client.
780
+ let userAgent: string | undefined
781
+ if (typeof page.evaluate === 'function') {
782
+ try {
783
+ const ua = await page.evaluate('navigator.userAgent')
784
+ if (typeof ua === 'string' && ua !== '') userAgent = ua
785
+ } catch {
786
+ userAgent = undefined
787
+ }
788
+ }
789
+ const response = await page.request.get(url.toString(), {
790
+ headers: {
791
+ ...(userAgent === undefined ? {} : { 'user-agent': userAgent }),
792
+ accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
793
+ 'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8',
794
+ },
795
+ timeout: Math.min(deadline.remainingMs(), 15_000),
796
+ signal: deadline.signal,
797
+ })
798
+ const headers = response.headers()
799
+ const status = response.status()
800
+ if (status < 200 || status >= 300) return undefined
801
+ if (classifyChallengeResponse(status, headers) === 'challenge') return undefined
802
+ if (classifyContentType(headers['content-type']) !== 'html') return undefined
803
+ const html = await response.text()
804
+ if (classifyChallengeHtml(html) !== 'none') return undefined
805
+ if (!denoise) return capResult(url.toString(), status, { kind: 'html', content: html })
806
+ return denoiseResult(url.toString(), status, html)
807
+ } catch {
808
+ return undefined
617
809
  }
618
- const bounded = html.length > MAX_PIPELINE_INPUT_CHARS ? html.slice(0, MAX_PIPELINE_INPUT_CHARS) : html
619
- const { markdown } = htmlToMarkdown(bounded, finalUrl)
620
- const result = capResult(finalUrl, statusCode, { kind: 'text', content: markdown })
621
- return bounded !== html
622
- ? { ...result, truncated: true }
623
- : result
624
810
  }
625
811
 
626
812
  /**
@@ -631,6 +817,11 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
631
817
  * status-only 204/205 — {@link recoverUnrenderedStatus}), and the response
632
818
  * being a download the browser refuses to commit as a document
633
819
  * (`Content-Disposition: attachment` — {@link recoverDownloadResponse}).
820
+ * A TRANSIENT transport failure (`net::ERR_NETWORK_CHANGED` and family)
821
+ * is neither: it carries no response and no render decision, only a
822
+ * momentary host-network blip that killed this one navigation — so it is
823
+ * retried on the SAME page, bounded by {@link MAX_TRANSIENT_NAV_RETRIES}
824
+ * attempts, a short backoff, and the remaining waiting budget.
634
825
  *
635
826
  * @returns the response `goto` resolved with, plus `recovered` when the
636
827
  * navigation failure was recovered into a finished result the caller must
@@ -642,20 +833,44 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
642
833
  tracker: MainFrameTracker,
643
834
  deadline: Deadline,
644
835
  denoise: boolean,
645
- ): Promise<{ response: PlaywrightResponse | null; recovered?: WebFetchResult }> {
646
- try {
647
- const response = await page.goto(url.toString(), { waitUntil: 'domcontentloaded', timeout: deadline.remainingMs() })
648
- return { response }
649
- } catch (error) {
650
- // Our own deadline/abort closing the page mid-navigation is a
651
- // cancellation (translateError classifies it) — never a status fact.
652
- if (!deadline.signal.aborted) {
653
- const recovered = await recoverUnrenderedStatus(error, tracker, url, denoise)
654
- if (recovered !== undefined) return { response: null, recovered }
655
- const download = await recoverDownloadResponse(error, page, tracker, url, deadline, denoise)
656
- if (download !== undefined) return { response: null, recovered: download }
836
+ ): Promise<{ response: PlaywrightResponse | null; recovered?: WebFetchResult; partial?: boolean }> {
837
+ for (let attempt = 0; ; attempt++) {
838
+ try {
839
+ const response = await page.goto(url.toString(), { waitUntil: 'domcontentloaded', timeout: deadline.remainingMs() })
840
+ return { response }
841
+ } catch (error) {
842
+ // Our own deadline/abort closing the page mid-navigation is a
843
+ // cancellation (translateError classifies it) — never a status fact.
844
+ if (!deadline.signal.aborted) {
845
+ const recovered = await recoverUnrenderedStatus(error, tracker, url, denoise)
846
+ if (recovered !== undefined) return { response: null, recovered }
847
+ const download = await recoverDownloadResponse(error, page, tracker, url, deadline, denoise)
848
+ if (download !== undefined) return { response: null, recovered: download }
849
+ // The waiting budget ran out mid-load: once the document's response
850
+ // arrived the page already holds content, so stop the load and let
851
+ // the caller extract it instead of failing the whole fetch.
852
+ const arrived = tracker.last()
853
+ if (deadline.expired && arrived !== null) {
854
+ await stopLoading(page)
855
+ return { response: arrived, partial: true }
856
+ }
857
+ // The transport blip family: nothing about the PAGE died — the
858
+ // host's network shifted under the browser and Chromium aborted
859
+ // the in-flight navigation. Retry on the same page while the
860
+ // attempt cap, the backoff, and the remaining budget allow; the
861
+ // reserve keeps enough budget for the settle/decode tail.
862
+ if (
863
+ attempt < MAX_TRANSIENT_NAV_RETRIES
864
+ && isTransientNavigationFailure(error)
865
+ && deadline.remainingMs() > TRANSIENT_RETRY_RESERVE_MS
866
+ ) {
867
+ const backoff = TRANSIENT_RETRY_BACKOFF_MS[Math.min(attempt, TRANSIENT_RETRY_BACKOFF_MS.length - 1)] ?? TRANSIENT_RETRY_BACKOFF_MS.at(-1) ?? 500
868
+ await abortableSleep(backoff, deadline.signal)
869
+ if (!deadline.signal.aborted) continue
870
+ }
871
+ }
872
+ throw error
657
873
  }
658
- throw error
659
874
  }
660
875
  }
661
876
 
@@ -715,6 +930,74 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
715
930
  return true
716
931
  }
717
932
  }
933
+
934
+ /**
935
+ * Best-effort settle for client-rendered content — a Lighthouse-style
936
+ * bounded network-quiet window replacing Playwright's `networkidle`
937
+ * (officially discouraged): worker-script fetches and cross-origin auth
938
+ * frames hold one request open on the page session FOREVER (their
939
+ * completion events are attributed to other CDP targets), so strict
940
+ * networkidle never arrives on exactly the modern pages the settle exists
941
+ * for, and the legacy wait burned its whole cap on them every fetch.
942
+ *
943
+ * Sequence: a `load` baseline (usually already past — domcontentloaded
944
+ * resolved earlier), then poll until no request has STARTED within
945
+ * {@link settleQuietMs} and every still-open request is older than
946
+ * {@link settleStaleMs} — a request that old is an attribution artifact
947
+ * or a long-poll whose completion no longer gates content. All waits
948
+ * share the {@link effectiveSettleMs} cap and the fetch deadline; a page
949
+ * that exposes no request events degrades to the legacy bounded
950
+ * networkidle wait. Never throws: any exit path keeps what the document
951
+ * already holds.
952
+ */
953
+ private async settleForContent(
954
+ page: PlaywrightPage,
955
+ deadline: Deadline,
956
+ config: ResolvedConfig,
957
+ activity: RequestActivity | undefined,
958
+ ): Promise<void> {
959
+ const beganAt = Date.now()
960
+ const cap = effectiveSettleMs(config)
961
+ if (cap <= 0) {
962
+ this.noteSettleOutcome('off', 0)
963
+ return
964
+ }
965
+ const endsAt = beganAt + Math.min(cap, deadline.remainingMs())
966
+ // The load event is the baseline quiet is measured against; usually it
967
+ // has already fired by the time the challenge loop released the page.
968
+ await page.waitForLoadState('load', { timeout: Math.max(1, endsAt - Date.now()) }).catch(() => {})
969
+ if (activity === undefined) {
970
+ // No request events on this surface: the legacy bounded wait.
971
+ await page.waitForLoadState('networkidle', { timeout: Math.max(1, endsAt - Date.now()) }).catch(() => {})
972
+ this.noteSettleOutcome('fallback', Date.now() - beganAt)
973
+ return
974
+ }
975
+ for (;;) {
976
+ if (deadline.signal.aborted) {
977
+ this.noteSettleOutcome('aborted', Date.now() - beganAt)
978
+ return
979
+ }
980
+ if (Date.now() >= endsAt) {
981
+ this.noteSettleOutcome('cap', Date.now() - beganAt)
982
+ return
983
+ }
984
+ if (activity.idle(this.settleQuietMs, this.settleStaleMs)) {
985
+ this.noteSettleOutcome(activity.openCount() > 0 ? 'stale-discounted' : 'quiet', Date.now() - beganAt)
986
+ return
987
+ }
988
+ await sleep(Math.min(this.settlePollMs, Math.max(1, endsAt - Date.now())))
989
+ }
990
+ }
991
+
992
+ /**
993
+ * Settle outcome observation hook — silent in production by design (the
994
+ * settle is best-effort), but subclasses (tests, diagnostics) override it
995
+ * to see which exit path every fetch took: `off` (config), `quiet`
996
+ * (network went quiet), `stale-discounted` (only stale requests remained
997
+ * open), `cap` (window exhausted), `fallback` (no request events — the
998
+ * legacy networkidle wait ran), `aborted` (fetch deadline fired).
999
+ */
1000
+ protected noteSettleOutcome(outcome: 'off' | 'quiet' | 'stale-discounted' | 'cap' | 'fallback' | 'aborted', elapsedMs: number): void {}
718
1001
  }
719
1002
 
720
1003
  /** Resolve after `ms` — the bounded wait's inter-poll nap. */
@@ -722,6 +1005,39 @@ function sleep(ms: number): Promise<void> {
722
1005
  return new Promise(resolve => { setTimeout(resolve, ms) })
723
1006
  }
724
1007
 
1008
+ /**
1009
+ * Best-effort `window.stop()` on a page whose load outran the fetch budget:
1010
+ * the parser and pending subresources halt, so the DOM read next is what
1011
+ * arrived rather than a moving target. Bounded; failures are ignored.
1012
+ */
1013
+ async function stopLoading(page: PlaywrightPage): Promise<void> {
1014
+ if (page.evaluate === undefined) return
1015
+ let timer: ReturnType<typeof setTimeout> | undefined
1016
+ await Promise.race([
1017
+ page.evaluate('window.stop()').catch(() => {}),
1018
+ new Promise<void>(resolve => { timer = setTimeout(resolve, STOP_LOADING_GRACE_MS) }),
1019
+ ])
1020
+ if (timer !== undefined) clearTimeout(timer)
1021
+ }
1022
+
1023
+ /** The rendered text of a stopped non-HTML document (its body's innerText). */
1024
+ async function renderedText(page: PlaywrightPage): Promise<string> {
1025
+ if (page.evaluate !== undefined) {
1026
+ try {
1027
+ const text = await page.evaluate('document.body ? document.body.innerText : ""')
1028
+ if (typeof text === 'string') return text
1029
+ } catch {
1030
+ // fall through to the serialized document
1031
+ }
1032
+ }
1033
+ return await page.content()
1034
+ }
1035
+
1036
+ /** Mark a result built from a document that never finished loading. */
1037
+ function flagPartial(result: WebFetchResult, partial: boolean): WebFetchResult {
1038
+ return partial ? { ...result, truncated: true } : result
1039
+ }
1040
+
725
1041
  /**
726
1042
  * Whether a response event belongs to the page's main frame as a document
727
1043
  * navigation — the filter the challenge wait uses to keep the LAST such
@@ -770,6 +1086,62 @@ function trackMainFrameResponses(page: PlaywrightPage): MainFrameTracker {
770
1086
  }
771
1087
  }
772
1088
 
1089
+ /**
1090
+ * The running record of the page's open requests, for the settle's
1091
+ * network-quiet detection. Attached BEFORE navigation so every request —
1092
+ * including ones that started before the settle begins — counts toward the
1093
+ * quiet window and staleness.
1094
+ */
1095
+ interface RequestActivity {
1096
+ /**
1097
+ * Whether the page is settle-idle: no request started within `quietMs`,
1098
+ * and every still-open request has been open at least `staleMs`.
1099
+ */
1100
+ idle(quietMs: number, staleMs: number): boolean
1101
+ /** How many requests are still open (labels the idle exit). */
1102
+ openCount(): number
1103
+ }
1104
+
1105
+ /**
1106
+ * Watch the page for request lifecycle events so the settle can decide
1107
+ * network quietness itself. `undefined` when the page exposes no request
1108
+ * events (minimal fakes): the settle then falls back to the legacy bounded
1109
+ * networkidle wait.
1110
+ *
1111
+ * @param page - the fetch's tab.
1112
+ * @returns the open-request tracker, or undefined on an unsupported surface.
1113
+ */
1114
+ function trackRequestActivity(page: PlaywrightPage): RequestActivity | undefined {
1115
+ if (typeof page.on !== 'function') return undefined
1116
+ const opened = new Map<PlaywrightWebRequest, number>()
1117
+ let lastStart = Date.now()
1118
+ const onStarted = (request: PlaywrightWebRequest): void => {
1119
+ opened.set(request, Date.now())
1120
+ lastStart = Date.now()
1121
+ }
1122
+ const onSettled = (request: PlaywrightWebRequest): void => {
1123
+ opened.delete(request)
1124
+ }
1125
+ try {
1126
+ page.on('request', onStarted)
1127
+ page.on('requestfinished', onSettled)
1128
+ page.on('requestfailed', onSettled)
1129
+ } catch {
1130
+ return undefined
1131
+ }
1132
+ return {
1133
+ idle: (quietMs, staleMs) => {
1134
+ const now = Date.now()
1135
+ if (now - lastStart < quietMs) return false
1136
+ for (const startedAt of opened.values()) {
1137
+ if (now - startedAt < staleMs) return false
1138
+ }
1139
+ return true
1140
+ },
1141
+ openCount: () => opened.size,
1142
+ }
1143
+ }
1144
+
773
1145
  /**
774
1146
  * Whether a `goto` throw is Chromium refusing to RENDER a response it
775
1147
  * received — not a transport failure. Two net errors, both documented as
@@ -794,6 +1166,48 @@ function isUnrenderedStatusFailure(error: unknown, status: number): boolean {
794
1166
  return message.includes('ERR_ABORTED') && (status === 204 || status === 205)
795
1167
  }
796
1168
 
1169
+ /**
1170
+ * Chromium net errors a plain re-navigation can plausibly fix — the
1171
+ * transport-layer BLIPS, not render decisions (those are recovered above)
1172
+ * and not `ERR_ABORTED` (deliberately excluded: that code is overloaded with
1173
+ * deterministic shapes — SPA cancels, 204/205 aborts, downloads — which the
1174
+ * recoveries already own). `ERR_NETWORK_CHANGED` is the measured badcase:
1175
+ * the host's network configuration shifted under the browser mid-navigation
1176
+ * (route/interface/DNS/proxy change, VPN or DHCP flap, a Docker
1177
+ * embedded-DNS rewrite) and Chromium's NetworkChangeNotifier killed the
1178
+ * in-flight request; the sibling codes are the same story on a different
1179
+ * layer (DNS blips, TCP resets and idle breaks, brief disconnects).
1180
+ * Playwright's own navigation timeouts never match: they reject with a
1181
+ * `TimeoutError` ("Timeout Nms exceeded"), not a `net::ERR_*` code.
1182
+ */
1183
+ const TRANSIENT_NET_ERROR_RE = /net::ERR_(NETWORK_CHANGED|INTERNET_DISCONNECTED|CONNECTION_RESET|CONNECTION_CLOSED|CONNECTION_TIMED_OUT|TIMED_OUT|EMPTY_RESPONSE|NAME_NOT_RESOLVED|NAME_RESOLUTION_FAILED|ADDRESS_UNREACHABLE)\b/
1184
+
1185
+ /**
1186
+ * Whether a `goto` throw is one of the transient transport failures a
1187
+ * bounded same-page retry can recover (see {@link TRANSIENT_NET_ERROR_RE}).
1188
+ *
1189
+ * @param error - the value `page.goto` rejected with.
1190
+ * @returns true when the navigation died to a momentary network blip.
1191
+ */
1192
+ function isTransientNavigationFailure(error: unknown): boolean {
1193
+ return error instanceof Error && TRANSIENT_NET_ERROR_RE.test(error.message)
1194
+ }
1195
+
1196
+ /**
1197
+ * Sleep for `ms`, ending EARLY when `signal` aborts: the transient-retry
1198
+ * backoff must never outlive the fetch — a deadline firing mid-backoff
1199
+ * resolves immediately so the caller re-checks the signal and rethrows the
1200
+ * original failure for the timeout/abort classification.
1201
+ */
1202
+ function abortableSleep(ms: number, signal: AbortSignal): Promise<void> {
1203
+ return new Promise(resolve => {
1204
+ if (signal.aborted) return resolve()
1205
+ const onAbort = () => { clearTimeout(timer); resolve() }
1206
+ const timer = setTimeout(() => { signal.removeEventListener('abort', onAbort); resolve() }, ms)
1207
+ signal.addEventListener('abort', onAbort, { once: true })
1208
+ })
1209
+ }
1210
+
797
1211
  /**
798
1212
  * Recover the response behind a status-only navigation failure into the
799
1213
  * result the web seam's contract demands — "a successful network fetch of a
@@ -848,8 +1262,7 @@ async function recoverUnrenderedStatus(
848
1262
  return capResult(finalUrl, status, { kind: 'text', content })
849
1263
  }
850
1264
  if (!denoise) return capResult(finalUrl, status, { kind: 'html', content })
851
- const { markdown } = htmlToMarkdown(content, finalUrl)
852
- return capResult(finalUrl, status, { kind: 'text', content: markdown })
1265
+ return denoiseResult(finalUrl, status, content)
853
1266
  }
854
1267
 
855
1268
  /**
@@ -961,8 +1374,7 @@ async function recoverDownloadResponse(
961
1374
  return capResult(finalUrl, statusCode, { kind: 'text', content })
962
1375
  }
963
1376
  if (!denoise) return capResult(finalUrl, statusCode, { kind: 'html', content })
964
- const { markdown } = htmlToMarkdown(content, finalUrl)
965
- return capResult(finalUrl, statusCode, { kind: 'text', content: markdown })
1377
+ return denoiseResult(finalUrl, statusCode, content)
966
1378
  }
967
1379
 
968
1380
  /** Time kept back from the PDF page walk for layout and result assembly. */
@@ -1077,6 +1489,20 @@ async function stripScriptingElements(page: PlaywrightPage): Promise<void> {
1077
1489
  }
1078
1490
  }
1079
1491
 
1492
+ /**
1493
+ * Denoise an HTML body into a capped text result. The synchronous pipeline
1494
+ * builds several DOMs per page, so its input is bounded on EVERY path that
1495
+ * reaches it (live render, unrenderable-loader fallback, refetch): an
1496
+ * uncapped 5MB API-reference page needs >4GB of heap. A cut input flags
1497
+ * the result truncated.
1498
+ */
1499
+ function denoiseResult(url: string, statusCode: number, html: string): WebFetchResult {
1500
+ const bounded = html.length > MAX_PIPELINE_INPUT_CHARS ? html.slice(0, MAX_PIPELINE_INPUT_CHARS) : html
1501
+ const { markdown } = htmlToMarkdown(bounded, url)
1502
+ const result = capResult(url, statusCode, { kind: 'text', content: markdown })
1503
+ return { ...result, truncated: result.truncated || bounded !== html }
1504
+ }
1505
+
1080
1506
  /** Apply the decoded-body cap and flag the cut. */
1081
1507
  function capResult(url: string, statusCode: number, body: { kind: 'html' | 'text'; content: string }): WebFetchResult {
1082
1508
  const truncated = body.content.length > MAX_BODY_CHARS