dsh-web-fetch-playwright 0.2.20 → 0.2.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/provider.ts CHANGED
@@ -55,9 +55,14 @@
55
55
  * behavior). The wait tracks the LAST main-frame navigation response (the
56
56
  * real page reloads in after the challenge clears) and watches the live DOM
57
57
  * so SPA-style clears are caught too. It never clicks, never injects
58
- * answers, never touches cookies itself; when the budget runs out the fetch
59
- * fails with {@link WEB_FETCH_CHALLENGE_CODE} instead of returning the
60
- * interstitial as content.
58
+ * answers, never touches cookies itself; when the budget runs out, ONE
59
+ * plain-HTTP refetch of the same URL through the page's request context
60
+ * (same cookie jar, the browser's UA, none of Chromium's render decisions
61
+ * — `challengeHttpFallback`, default on) tries to retrieve the real
62
+ * document, gated by the same response-level and content-level challenge
63
+ * classifiers so an interstitial can never pass as content. When that too
64
+ * fails, the fetch fails with {@link WEB_FETCH_CHALLENGE_CODE} instead of
65
+ * returning the interstitial as content.
61
66
  *
62
67
  * Deep capture (viewport-virtualized feeds and app-shell documents): feed-
63
68
  * shaped pages — Discourse topic streams, comment threads, timelines —
@@ -84,6 +89,17 @@
84
89
  * degrades to the legacy capture (see `src/autoscroll.ts` for the measured
85
90
  * linux.do and Feishu badcases).
86
91
  *
92
+ * Deadline (soft, then hard): each fetch has a {@link DEFAULT_TIMEOUT_MS}
93
+ * budget for WAITING — navigation, settle, challenge wait, scroll walk. When
94
+ * it runs out the page is NOT torn down: every wait is bounded by the same
95
+ * budget and simply ends, a navigation still loading is stopped
96
+ * (`window.stop()`) once its main-frame response has arrived, and whatever
97
+ * the page holds by then goes through the normal extraction — a slow or
98
+ * endless page still hands the agent the content it already rendered
99
+ * (flagged `truncated` when the document never finished loading). Only when
100
+ * nothing arrived at all, or extraction itself hangs past
101
+ * {@link EXTRACT_GRACE_MS}, does the fetch fail with `WEB_FETCH_TIMEOUT`.
102
+ *
87
103
  * @module dsh-web-fetch-playwright/provider
88
104
  */
89
105
 
@@ -91,8 +107,8 @@ import { WebError } from '@deepseek-ai/dsh-web'
91
107
  import type { WebFetchProvider, WebFetchRequest, WebFetchResult } from '@deepseek-ai/dsh-web'
92
108
  import { CHALLENGE_DOM_PROBE, CHALLENGE_FINISH_RESERVE_MS, CHALLENGE_POLL_INTERVAL_MS, classifyChallengeHtml, classifyChallengeResponse, isChallengeCompatibleResponse } from './challenge.ts'
93
109
  import type { ChallengeVerdict } from './challenge.ts'
94
- import { deepCaptureWalk, SCROLL_FINISH_RESERVE_MS } from './autoscroll.ts'
95
- import { DEFAULT_MAX_CONCURRENCY_CDP, DEFAULT_MAX_CONCURRENCY_LOCAL, effectiveChallengeRetries, effectiveChallengeWaitMs, effectiveContextMode, effectiveMaxConcurrency, effectiveScrollBudgetMs, normalizeCdpEndpoint } from './config.ts'
110
+ import { deepCaptureWalk } from './autoscroll.ts'
111
+ import { DEFAULT_MAX_CONCURRENCY_CDP, DEFAULT_MAX_CONCURRENCY_LOCAL, effectiveChallengeHttpFallback, effectiveChallengeRetries, effectiveChallengeWaitMs, effectiveContextMode, effectiveMaxConcurrency, effectiveScrollBudgetMs, normalizeCdpEndpoint } from './config.ts'
96
112
  import type { ResolvedConfig } from './config.ts'
97
113
  import { CdpConnectionPool } from './cdp-pool.ts'
98
114
  import { htmlToMarkdown } from './markdown.ts'
@@ -128,10 +144,23 @@ const MAX_PIPELINE_INPUT_CHARS = 2_000_000
128
144
  * budget to render after dequeue — and failing fast tells the caller to
129
145
  * retry or raise `maxConcurrency` instead of hanging until an abort.
130
146
  */
131
- const QUEUE_TIMEOUT_MS = 20_000
147
+ const QUEUE_TIMEOUT_MS = 10_000
148
+
149
+ /**
150
+ * Per-fetch waiting budget (ms): when it runs out the fetch stops waiting
151
+ * for the page and extracts what it already holds (see the module docs).
152
+ */
153
+ const DEFAULT_TIMEOUT_MS = 25_000
154
+
155
+ /**
156
+ * Hard backstop (ms) after {@link DEFAULT_TIMEOUT_MS} for reading and
157
+ * decoding what the page holds; only past it is the tab closed and the fetch
158
+ * failed. Budget plus grace stays inside the tool layer's 60s.
159
+ */
160
+ const EXTRACT_GRACE_MS = 10_000
132
161
 
133
- /** Default per-fetch budget (ms), inside the tool layer's 60s. */
134
- const DEFAULT_TIMEOUT_MS = 45_000
162
+ /** Bound (ms) on the best-effort `window.stop()` of a still-loading page. */
163
+ const STOP_LOADING_GRACE_MS = 1_000
135
164
 
136
165
  /** Best-effort post-DOM settle wait (ms) so SPA content can finish rendering. */
137
166
  const SETTLE_MS = 5_000
@@ -163,9 +192,13 @@ export interface BrowserSession {
163
192
  }
164
193
 
165
194
  /**
166
- * A single abort deadline composing the caller's signal with this provider's
167
- * time budget. The timeout reason is kept apart from an outer abort so the
168
- * error translation can pick `WEB_FETCH_TIMEOUT` over `WEB_ABORTED`.
195
+ * The fetch's two-stage deadline composed with the caller's signal. The SOFT
196
+ * stage (`timeoutMs`) only bounds waiting: {@link remainingMs} counts down to
197
+ * it and {@link expired} reports it, so every wait ends there and the fetch
198
+ * extracts what the page holds. The HARD stage (`timeoutMs + graceMs`) aborts
199
+ * {@link signal}, which closes the page. The timeout reason is kept apart
200
+ * from an outer abort so the error translation can pick `WEB_FETCH_TIMEOUT`
201
+ * over `WEB_ABORTED`.
169
202
  */
170
203
  class Deadline {
171
204
  readonly signal: AbortSignal
@@ -173,13 +206,13 @@ class Deadline {
173
206
  private readonly expiresAt: number
174
207
  private timedOut = false
175
208
 
176
- constructor(outer: AbortSignal | undefined, timeoutMs: number) {
209
+ constructor(outer: AbortSignal | undefined, timeoutMs: number, graceMs = 0) {
177
210
  this.signal = this.controller.signal
178
211
  this.expiresAt = Date.now() + timeoutMs
179
212
  const timer = setTimeout(() => {
180
213
  this.timedOut = true
181
214
  this.controller.abort(new Error('playwright fetch deadline'))
182
- }, timeoutMs)
215
+ }, timeoutMs + graceMs)
183
216
  if (outer !== undefined) {
184
217
  if (outer.aborted) this.controller.abort(outer.reason)
185
218
  else outer.addEventListener('abort', () => { this.controller.abort(outer.reason) }, { once: true })
@@ -187,12 +220,17 @@ class Deadline {
187
220
  this.signal.addEventListener('abort', () => { clearTimeout(timer) }, { once: true })
188
221
  }
189
222
 
190
- /** Whether OUR timer fired (vs an outer cancellation). */
223
+ /** Whether OUR hard timer fired (vs an outer cancellation). */
191
224
  get isTimeout(): boolean {
192
225
  return this.timedOut
193
226
  }
194
227
 
195
- /** Milliseconds left on the budget, floored at 1 for Playwright options. */
228
+ /** Whether the soft (waiting) budget has run out. */
229
+ get expired(): boolean {
230
+ return Date.now() >= this.expiresAt
231
+ }
232
+
233
+ /** Milliseconds left on the waiting budget, floored at 1 for Playwright options. */
196
234
  remainingMs(): number {
197
235
  return Math.max(1, this.expiresAt - Date.now())
198
236
  }
@@ -324,6 +362,9 @@ function translateError(error: unknown, deadline: Deadline): WebError {
324
362
  return new WebError('web fetch aborted', 'WEB_ABORTED', { cause: error })
325
363
  }
326
364
  if (error instanceof WebError) return error
365
+ // Past the waiting budget with nothing salvageable (no response arrived,
366
+ // or a bounded wait's own timeout surfaced): the budget is the cause.
367
+ if (deadline.expired) return new WebError('playwright web fetch timed out before the page delivered any content', 'WEB_FETCH_TIMEOUT', { cause: error })
327
368
  return new WebError(`playwright web fetch failed: ${String(error instanceof Error ? error.message : error)}`, 'WEB_PROVIDER_ERROR', { cause: error })
328
369
  }
329
370
 
@@ -337,6 +378,12 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
337
378
 
338
379
  private readonly semaphore = new Semaphore(DEFAULT_MAX_CONCURRENCY_LOCAL)
339
380
 
381
+ /** Waiting budget per fetch (ms); code-level, overridable by subclasses (tests). */
382
+ protected readonly fetchTimeoutMs: number = DEFAULT_TIMEOUT_MS
383
+
384
+ /** Extraction grace after the waiting budget (ms); code-level, overridable by subclasses. */
385
+ protected readonly extractGraceMs: number = EXTRACT_GRACE_MS
386
+
340
387
  /** Shared CDP connection; injectable so the suite can substitute a fake. */
341
388
  protected readonly cdpPool: CdpConnectionPool
342
389
 
@@ -382,7 +429,7 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
382
429
  }
383
430
  }
384
431
  const url = validateFetchUrl(request.url)
385
- const deadline = new Deadline(signal, DEFAULT_TIMEOUT_MS)
432
+ const deadline = new Deadline(signal, this.fetchTimeoutMs, this.extractGraceMs)
386
433
 
387
434
  let session: BrowserSession | undefined
388
435
  let acquired = false
@@ -394,6 +441,7 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
394
441
  acquired = true
395
442
  session = await this.openSession(config, deadline, sessionId)
396
443
  if (deadline.signal.aborted) throw new Error('web fetch aborted during session setup')
444
+ if (deadline.expired) throw new Error('the fetch budget ran out during session setup')
397
445
  // An aborted deadline must also interrupt Playwright's own waits:
398
446
  // closing the page rejects every pending operation on it (and, for
399
447
  // fetch-owned contexts, the context close that follows takes the rest).
@@ -510,6 +558,9 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
510
558
  const first = await this.navigate(page, url, tracker, deadline, config.denoise)
511
559
  if (first.recovered !== undefined) return first.recovered
512
560
  let response = first.response
561
+ // The document never finished loading inside the budget: extract what
562
+ // arrived and flag the result as incomplete.
563
+ let partial = first.partial === true
513
564
  tracker.seed(response)
514
565
 
515
566
  let challengeEntryResponse: PlaywrightResponse | null = null
@@ -536,7 +587,18 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
536
587
  try { chained = classifyChallengeHtml(await page.content()) === 'challenge' } catch { chained = false }
537
588
  if (!chained) break
538
589
  }
539
- if (--attemptsLeft <= 0) {
590
+ // No budget left for another window: a re-navigation could only
591
+ // land on the interstitial again.
592
+ if (--attemptsLeft <= 0 || deadline.expired) {
593
+ // The dead-end refetch: many Cloudflare edges serve the real
594
+ // document to plain-HTTP clients (same cookies, no render
595
+ // decision) while withholding the interactive challenge from
596
+ // THIS browser session — measured live on v2ex.com, where the
597
+ // browser drew an empty widget frame and curl drew the article.
598
+ if (effectiveChallengeHttpFallback(config)) {
599
+ const fallback = await this.challengeHttpFallback(page, url, deadline, config.denoise)
600
+ if (fallback !== undefined) return fallback
601
+ }
540
602
  const lastStatus = (tracker.last() ?? response)?.status()
541
603
  throw new WebError(
542
604
  `the site kept serving a Cloudflare challenge (last status ${lastStatus === undefined ? 'unknown' : String(lastStatus)}) for up to ${String(challengeWaitMs)}ms across ${String(effectiveChallengeRetries(config) + 1)} attempt(s); the browser did not clear it naturally — retry later, raise challengeWaitMs, or use a profile whose browser already holds clearance`,
@@ -549,6 +611,7 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
549
611
  const retry = await this.navigate(page, url, tracker, deadline, config.denoise)
550
612
  if (retry.recovered !== undefined) return retry.recovered
551
613
  response = retry.response
614
+ partial = retry.partial === true
552
615
  tracker.seed(response)
553
616
  }
554
617
  }
@@ -577,8 +640,12 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
577
640
 
578
641
  // Non-HTML decodes straight from the response body; no denoise applies.
579
642
  if (kind === 'text') {
580
- const text = finalResponse !== null ? await finalResponse.text() : await page.content()
581
- return capResult(finalUrl, statusCode, { kind: 'text', content: text })
643
+ // A stopped load has no complete response body to read; the page's
644
+ // rendered text is what arrived.
645
+ const text = partial
646
+ ? await renderedText(page)
647
+ : finalResponse !== null ? await finalResponse.text() : await page.content()
648
+ return flagPartial(capResult(finalUrl, statusCode, { kind: 'text', content: text }), partial)
582
649
  }
583
650
 
584
651
  // Best-effort settle for client-rendered content; a timeout just keeps
@@ -589,12 +656,11 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
589
656
  // their off-window content at rest — see `src/autoscroll.ts`): a bounded
590
657
  // walk stashes what each rendered window shows and, when it restored
591
658
  // anything, hands back its own merged, already-stripped serialization.
592
- // HTML documents only, inside the deadline with a decode reserve,
593
- // silently degrading to the legacy capture; `scrollBudgetMs: 0` skips it.
594
- const scrollBudgetMs = Math.min(
595
- effectiveScrollBudgetMs(config),
596
- deadline.remainingMs() - SCROLL_FINISH_RESERVE_MS,
597
- )
659
+ // HTML documents only, ending at the waiting budget at the latest — the
660
+ // walk then hands back what it stashed so far and extraction runs in
661
+ // the grace window — silently degrading to the legacy capture;
662
+ // `scrollBudgetMs: 0` skips it, as does a budget already spent.
663
+ const scrollBudgetMs = deadline.expired ? 0 : Math.min(effectiveScrollBudgetMs(config), deadline.remainingMs())
598
664
  const captured = scrollBudgetMs > 0 ? await deepCaptureWalk(page, scrollBudgetMs) : undefined
599
665
 
600
666
  // Serialization safety for the live read: the document is about to be
@@ -613,14 +679,56 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
613
679
  if (!config.denoise) {
614
680
  // The tool layer's own turndown renders raw HTML; the checkbox only
615
681
  // governs the Readability/DOMPurify stage this provider owns.
616
- return capResult(finalUrl, statusCode, { kind: 'html', content: html })
682
+ return flagPartial(capResult(finalUrl, statusCode, { kind: 'html', content: html }), partial)
683
+ }
684
+ return flagPartial(denoiseResult(finalUrl, statusCode, html), partial)
685
+ }
686
+
687
+ /**
688
+ * The challenge dead-end's plain-HTTP refetch: one request through the
689
+ * page's own request context (the SAME cookie jar — any clearance the
690
+ * profile holds rides along — but none of Chromium's render decisions),
691
+ * wearing the browser's own User-Agent. Returns the finished result when
692
+ * the answer is a NON-challenge HTML document, undefined otherwise (the
693
+ * caller throws the honest WEB_FETCH_CHALLENGE). A challenge served to
694
+ * this request too, a non-HTML body, or any transport failure keeps the
695
+ * browser-only verdict.
696
+ */
697
+ private async challengeHttpFallback(page: PlaywrightPage, url: URL, deadline: Deadline, denoise: boolean): Promise<WebFetchResult | undefined> {
698
+ if (page.request === undefined || deadline.expired || deadline.signal.aborted) return undefined
699
+ try {
700
+ // A CDP-attached browser's request context does not inherit the
701
+ // browser's UA — send it explicitly so the edge sees one client.
702
+ let userAgent: string | undefined
703
+ if (typeof page.evaluate === 'function') {
704
+ try {
705
+ const ua = await page.evaluate('navigator.userAgent')
706
+ if (typeof ua === 'string' && ua !== '') userAgent = ua
707
+ } catch {
708
+ userAgent = undefined
709
+ }
710
+ }
711
+ const response = await page.request.get(url.toString(), {
712
+ headers: {
713
+ ...(userAgent === undefined ? {} : { 'user-agent': userAgent }),
714
+ accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
715
+ 'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8',
716
+ },
717
+ timeout: Math.min(deadline.remainingMs(), 15_000),
718
+ signal: deadline.signal,
719
+ })
720
+ const headers = response.headers()
721
+ const status = response.status()
722
+ if (status < 200 || status >= 300) return undefined
723
+ if (classifyChallengeResponse(status, headers) === 'challenge') return undefined
724
+ if (classifyContentType(headers['content-type']) !== 'html') return undefined
725
+ const html = await response.text()
726
+ if (classifyChallengeHtml(html) !== 'none') return undefined
727
+ if (!denoise) return capResult(url.toString(), status, { kind: 'html', content: html })
728
+ return denoiseResult(url.toString(), status, html)
729
+ } catch {
730
+ return undefined
617
731
  }
618
- const bounded = html.length > MAX_PIPELINE_INPUT_CHARS ? html.slice(0, MAX_PIPELINE_INPUT_CHARS) : html
619
- const { markdown } = htmlToMarkdown(bounded, finalUrl)
620
- const result = capResult(finalUrl, statusCode, { kind: 'text', content: markdown })
621
- return bounded !== html
622
- ? { ...result, truncated: true }
623
- : result
624
732
  }
625
733
 
626
734
  /**
@@ -642,7 +750,7 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
642
750
  tracker: MainFrameTracker,
643
751
  deadline: Deadline,
644
752
  denoise: boolean,
645
- ): Promise<{ response: PlaywrightResponse | null; recovered?: WebFetchResult }> {
753
+ ): Promise<{ response: PlaywrightResponse | null; recovered?: WebFetchResult; partial?: boolean }> {
646
754
  try {
647
755
  const response = await page.goto(url.toString(), { waitUntil: 'domcontentloaded', timeout: deadline.remainingMs() })
648
756
  return { response }
@@ -654,6 +762,14 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
654
762
  if (recovered !== undefined) return { response: null, recovered }
655
763
  const download = await recoverDownloadResponse(error, page, tracker, url, deadline, denoise)
656
764
  if (download !== undefined) return { response: null, recovered: download }
765
+ // The waiting budget ran out mid-load: once the document's response
766
+ // arrived the page already holds content, so stop the load and let
767
+ // the caller extract it instead of failing the whole fetch.
768
+ const arrived = tracker.last()
769
+ if (deadline.expired && arrived !== null) {
770
+ await stopLoading(page)
771
+ return { response: arrived, partial: true }
772
+ }
657
773
  }
658
774
  throw error
659
775
  }
@@ -722,6 +838,39 @@ function sleep(ms: number): Promise<void> {
722
838
  return new Promise(resolve => { setTimeout(resolve, ms) })
723
839
  }
724
840
 
841
+ /**
842
+ * Best-effort `window.stop()` on a page whose load outran the fetch budget:
843
+ * the parser and pending subresources halt, so the DOM read next is what
844
+ * arrived rather than a moving target. Bounded; failures are ignored.
845
+ */
846
+ async function stopLoading(page: PlaywrightPage): Promise<void> {
847
+ if (page.evaluate === undefined) return
848
+ let timer: ReturnType<typeof setTimeout> | undefined
849
+ await Promise.race([
850
+ page.evaluate('window.stop()').catch(() => {}),
851
+ new Promise<void>(resolve => { timer = setTimeout(resolve, STOP_LOADING_GRACE_MS) }),
852
+ ])
853
+ if (timer !== undefined) clearTimeout(timer)
854
+ }
855
+
856
+ /** The rendered text of a stopped non-HTML document (its body's innerText). */
857
+ async function renderedText(page: PlaywrightPage): Promise<string> {
858
+ if (page.evaluate !== undefined) {
859
+ try {
860
+ const text = await page.evaluate('document.body ? document.body.innerText : ""')
861
+ if (typeof text === 'string') return text
862
+ } catch {
863
+ // fall through to the serialized document
864
+ }
865
+ }
866
+ return await page.content()
867
+ }
868
+
869
+ /** Mark a result built from a document that never finished loading. */
870
+ function flagPartial(result: WebFetchResult, partial: boolean): WebFetchResult {
871
+ return partial ? { ...result, truncated: true } : result
872
+ }
873
+
725
874
  /**
726
875
  * Whether a response event belongs to the page's main frame as a document
727
876
  * navigation — the filter the challenge wait uses to keep the LAST such
@@ -848,8 +997,7 @@ async function recoverUnrenderedStatus(
848
997
  return capResult(finalUrl, status, { kind: 'text', content })
849
998
  }
850
999
  if (!denoise) return capResult(finalUrl, status, { kind: 'html', content })
851
- const { markdown } = htmlToMarkdown(content, finalUrl)
852
- return capResult(finalUrl, status, { kind: 'text', content: markdown })
1000
+ return denoiseResult(finalUrl, status, content)
853
1001
  }
854
1002
 
855
1003
  /**
@@ -961,8 +1109,7 @@ async function recoverDownloadResponse(
961
1109
  return capResult(finalUrl, statusCode, { kind: 'text', content })
962
1110
  }
963
1111
  if (!denoise) return capResult(finalUrl, statusCode, { kind: 'html', content })
964
- const { markdown } = htmlToMarkdown(content, finalUrl)
965
- return capResult(finalUrl, statusCode, { kind: 'text', content: markdown })
1112
+ return denoiseResult(finalUrl, statusCode, content)
966
1113
  }
967
1114
 
968
1115
  /** Time kept back from the PDF page walk for layout and result assembly. */
@@ -1077,6 +1224,20 @@ async function stripScriptingElements(page: PlaywrightPage): Promise<void> {
1077
1224
  }
1078
1225
  }
1079
1226
 
1227
+ /**
1228
+ * Denoise an HTML body into a capped text result. The synchronous pipeline
1229
+ * builds several DOMs per page, so its input is bounded on EVERY path that
1230
+ * reaches it (live render, unrenderable-loader fallback, refetch): an
1231
+ * uncapped 5MB API-reference page needs >4GB of heap. A cut input flags
1232
+ * the result truncated.
1233
+ */
1234
+ function denoiseResult(url: string, statusCode: number, html: string): WebFetchResult {
1235
+ const bounded = html.length > MAX_PIPELINE_INPUT_CHARS ? html.slice(0, MAX_PIPELINE_INPUT_CHARS) : html
1236
+ const { markdown } = htmlToMarkdown(bounded, url)
1237
+ const result = capResult(url, statusCode, { kind: 'text', content: markdown })
1238
+ return { ...result, truncated: result.truncated || bounded !== html }
1239
+ }
1240
+
1080
1241
  /** Apply the decoded-body cap and flag the cut. */
1081
1242
  function capResult(url: string, statusCode: number, body: { kind: 'html' | 'text'; content: string }): WebFetchResult {
1082
1243
  const truncated = body.content.length > MAX_BODY_CHARS