dsh-web-fetch-playwright 0.2.4 → 0.2.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/provider.ts CHANGED
@@ -22,21 +22,44 @@
22
22
  * browser can reach. Profile mode additionally acts WITH the remote
23
23
  * browser's logged-in sessions (see the README's risk notes).
24
24
  *
25
+ * Cloudflare challenges (issue #2): when a navigation lands on a challenge
26
+ * interstitial, the fetch waits — on the SAME page and in the SAME browser
27
+ * context, so the browser's natural verification and any clearance cookies
28
+ * it earns apply — for a bounded, configurable window
29
+ * (`challengeWaitMs`, default 15s; 0 restores the legacy first-response
30
+ * behavior). The wait tracks the LAST main-frame navigation response (the
31
+ * real page reloads in after the challenge clears) and watches the live DOM
32
+ * so SPA-style clears are caught too. It never clicks, never injects
33
+ * answers, never touches cookies itself; when the budget runs out the fetch
34
+ * fails with {@link WEB_FETCH_CHALLENGE_CODE} instead of returning the
35
+ * interstitial as content.
36
+ *
25
37
  * @module dsh-web-fetch-playwright/provider
26
38
  */
27
39
 
28
40
  import { WebError } from '@deepseek-ai/dsh-web'
29
41
  import type { WebFetchProvider, WebFetchRequest, WebFetchResult } from '@deepseek-ai/dsh-web'
30
- import { DEFAULT_MAX_CONCURRENCY_CDP, DEFAULT_MAX_CONCURRENCY_LOCAL, effectiveContextMode, effectiveMaxConcurrency, normalizeCdpEndpoint } from './config.ts'
42
+ import { CHALLENGE_DOM_PROBE, CHALLENGE_FINISH_RESERVE_MS, CHALLENGE_POLL_INTERVAL_MS, classifyChallengeHtml, classifyChallengeResponse, isChallengeCompatibleResponse } from './challenge.ts'
43
+ import type { ChallengeVerdict } from './challenge.ts'
44
+ import { DEFAULT_MAX_CONCURRENCY_CDP, DEFAULT_MAX_CONCURRENCY_LOCAL, effectiveChallengeRetries, effectiveChallengeWaitMs, effectiveContextMode, effectiveMaxConcurrency, normalizeCdpEndpoint } from './config.ts'
31
45
  import type { ResolvedConfig } from './config.ts'
32
46
  import { CdpConnectionPool } from './cdp-pool.ts'
33
47
  import { htmlToMarkdown } from './markdown.ts'
34
48
  import { resolveCdpBackend, resolvePlaywrightBackend } from './playwright-resolve.ts'
35
- import type { PlaywrightBrowser, PlaywrightContext, PlaywrightPage, PlaywrightRoute } from './types.ts'
49
+ import type { PlaywrightBrowser, PlaywrightContext, PlaywrightPage, PlaywrightResponse, PlaywrightRoute } from './types.ts'
36
50
 
37
51
  /** Stable id this provider registers under (the bundle patch pins it). */
38
52
  export const PLAYWRIGHT_FETCH_PROVIDER_ID = 'playwright'
39
53
 
54
+ /**
55
+ * Error code for a Cloudflare challenge the bounded natural wait could not
56
+ * clear. Provider-specific by design — the web seam's `code` is an open
57
+ * string and consumers must tolerate provider-specific codes — so callers
58
+ * can tell "the site challenged us and the browser did not pass" apart from
59
+ * a transport timeout or a provider bug.
60
+ */
61
+ export const WEB_FETCH_CHALLENGE_CODE = 'WEB_FETCH_CHALLENGE'
62
+
40
63
  /** Maximum accepted request URL length (http-provider parity). */
41
64
  const MAX_URL_LENGTH = 2048
42
65
 
@@ -378,7 +401,11 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
378
401
  }
379
402
  }
380
403
 
381
- /** Navigate, settle, and decode one URL inside an open session's tab. */
404
+ /**
405
+ * Navigate, settle, and decode one URL inside an open session's tab —
406
+ * waiting out any Cloudflare interstitial within the bounded challenge
407
+ * budget before the final document is read (see the class docs).
408
+ */
382
409
  private async retrieve(
383
410
  session: BrowserSession,
384
411
  url: URL,
@@ -386,18 +413,70 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
386
413
  deadline: Deadline,
387
414
  ): Promise<WebFetchResult> {
388
415
  const page = session.page
389
- const response = await page.goto(url.toString(), { waitUntil: 'domcontentloaded', timeout: deadline.remainingMs() })
416
+ const challengeWaitMs = effectiveChallengeWaitMs(config)
417
+ // Feature switch: 0 keeps the exact legacy (pre-0.2.5) behavior — the
418
+ // first response decides, no waiting — an escape hatch and the A/B
419
+ // baseline every test proves the bug against.
420
+ const tracker = challengeWaitMs > 0 ? trackMainFrameResponses(page) : undefined
421
+ let response = await page.goto(url.toString(), { waitUntil: 'domcontentloaded', timeout: deadline.remainingMs() })
422
+ tracker?.seed(response)
423
+
424
+ let challengeEntryResponse: PlaywrightResponse | null = null
425
+ if (challengeWaitMs > 0) {
426
+ let attemptsLeft = effectiveChallengeRetries(config) + 1
427
+ for (;;) {
428
+ const verdict = await this.verdictAfterLoad(page, tracker?.last() ?? response)
429
+ if (verdict === 'blocked') {
430
+ throw new WebError(
431
+ `the site hard-blocked this fetch at its Cloudflare edge (waiting cannot clear it): ${page.url()}`,
432
+ WEB_FETCH_CHALLENGE_CODE,
433
+ )
434
+ }
435
+ if (verdict !== 'challenge') break
436
+ challengeEntryResponse = tracker?.last() ?? response
437
+ const cleared = await this.waitForChallengeClear(page, deadline, challengeWaitMs)
438
+ if (cleared) {
439
+ // The settled document may be the next round of a CHAINED
440
+ // challenge (JS test → Turnstile interstitial): the probe can
441
+ // clear in the gap between rounds, so confirm on the settled
442
+ // DOM. Deliberately content-level — an SPA clear keeps the 403
443
+ // challenge response forever and must still pass here.
444
+ let chained = false
445
+ try { chained = classifyChallengeHtml(await page.content()) === 'challenge' } catch { chained = false }
446
+ if (!chained) break
447
+ }
448
+ if (--attemptsLeft <= 0) {
449
+ const lastStatus = (tracker?.last() ?? response)?.status()
450
+ throw new WebError(
451
+ `the site kept serving a Cloudflare challenge (last status ${lastStatus === undefined ? 'unknown' : String(lastStatus)}) for up to ${String(challengeWaitMs)}ms across ${String(effectiveChallengeRetries(config) + 1)} attempt(s); the browser did not clear it naturally — retry later, raise challengeWaitMs, or use a profile whose browser already holds clearance`,
452
+ WEB_FETCH_CHALLENGE_CODE,
453
+ )
454
+ }
455
+ // Same page, same context — after an expired window OR a chained
456
+ // round: any clearance cookies already earned stay in the jar for
457
+ // this one retry, then everything is torn down as usual.
458
+ response = await page.goto(url.toString(), { waitUntil: 'domcontentloaded', timeout: deadline.remainingMs() })
459
+ tracker?.seed(response)
460
+ }
461
+ }
390
462
 
391
- const kind = classifyContentType(response?.headers()['content-type'])
463
+ // Final document: the LAST main-frame response when the challenge wait
464
+ // ran (the real page reloads in), else the response goto returned.
465
+ const finalResponse = (challengeWaitMs > 0 ? tracker?.last() : undefined) ?? response
466
+ const kind = classifyContentType(finalResponse?.headers()['content-type'])
392
467
  if (kind === undefined) {
393
- throw new WebError(`unsupported content type "${response?.headers()['content-type'] ?? 'unknown'}"`, 'WEB_UNSUPPORTED_CONTENT_TYPE')
468
+ throw new WebError(`unsupported content type "${finalResponse?.headers()['content-type'] ?? 'unknown'}"`, 'WEB_UNSUPPORTED_CONTENT_TYPE')
394
469
  }
395
470
  const finalUrl = page.url()
471
+ // An SPA-style clear swaps the document without navigating: no new
472
+ // response exists to report, so the cleared document reads as served.
473
+ const clearedWithoutNavigation = challengeEntryResponse !== null && finalResponse === challengeEntryResponse
474
+ const statusCode = finalResponse !== null && !clearedWithoutNavigation ? finalResponse.status() : 200
396
475
 
397
476
  // Non-HTML decodes straight from the response body; no denoise applies.
398
477
  if (kind === 'text') {
399
- const text = response !== null ? await response.text() : await page.content()
400
- return capResult(finalUrl, response?.status() ?? 200, { kind: 'text', content: text })
478
+ const text = finalResponse !== null ? await finalResponse.text() : await page.content()
479
+ return capResult(finalUrl, statusCode, { kind: 'text', content: text })
401
480
  }
402
481
 
403
482
  // Best-effort settle for client-rendered content; a timeout just keeps
@@ -408,15 +487,149 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
408
487
  if (!config.denoise) {
409
488
  // The tool layer's own turndown renders raw HTML; the checkbox only
410
489
  // governs the Readability/DOMPurify stage this provider owns.
411
- return capResult(finalUrl, response?.status() ?? 200, { kind: 'html', content: html })
490
+ return capResult(finalUrl, statusCode, { kind: 'html', content: html })
412
491
  }
413
492
  const bounded = html.length > MAX_PIPELINE_INPUT_CHARS ? html.slice(0, MAX_PIPELINE_INPUT_CHARS) : html
414
493
  const { markdown } = htmlToMarkdown(bounded, finalUrl)
415
- const result = capResult(finalUrl, response?.status() ?? 200, { kind: 'text', content: markdown })
494
+ const result = capResult(finalUrl, statusCode, { kind: 'text', content: markdown })
416
495
  return bounded !== html
417
496
  ? { ...result, truncated: true }
418
497
  : result
419
498
  }
499
+
500
+ /**
501
+ * Whether the freshly loaded document is a Cloudflare interstitial: the
502
+ * response-level signals first (the documented `cf-mitigated` header, then
503
+ * the 403/503 + cloudflare fallback), then — gated behind
504
+ * {@link isChallengeCompatibleResponse}, because interstitials never ship
505
+ * a plain 200 — the localized title family and structural markers, which
506
+ * also recognize the hard-block page waiting can never clear.
507
+ */
508
+ private async verdictAfterLoad(page: PlaywrightPage, response: PlaywrightResponse | null): Promise<ChallengeVerdict> {
509
+ if (response !== null && classifyChallengeResponse(response.status(), response.headers()) === 'challenge') return 'challenge'
510
+ // Suspicion gate: the content-level fallback only runs on a
511
+ // challenge-compatible response, so a normal article cannot be misread
512
+ // as a challenge no matter what it quotes. A response-less navigation
513
+ // keeps the check — its headers are simply unavailable.
514
+ if (response !== null && !isChallengeCompatibleResponse(response.status(), response.headers())) return 'none'
515
+ // Only HTML documents can be interstitials; classifyContentType already
516
+ // reads a missing content type as html (a response-less navigation).
517
+ if (response === null || classifyContentType(response.headers()['content-type']) === 'html') {
518
+ try {
519
+ return classifyChallengeHtml(await page.content())
520
+ } catch {
521
+ return 'none' // a document we cannot read is not a challenge we can wait on
522
+ }
523
+ }
524
+ return 'none'
525
+ }
526
+
527
+ /**
528
+ * The bounded natural wait: poll the live document for challenge markers
529
+ * until they are gone (the user's real browser passed the verification and
530
+ * reloaded into the real page, or swapped it in SPA-style), the per-fetch
531
+ * wait budget runs out, or the fetch's deadline aborts. No clicks, no
532
+ * injected answers — the browser either clears it on its own or it does not.
533
+ *
534
+ * @returns true when the challenge cleared; false when the budget ran out.
535
+ */
536
+ private async waitForChallengeClear(page: PlaywrightPage, deadline: Deadline, challengeWaitMs: number): Promise<boolean> {
537
+ // Keep the settle + decode tail inside the fetch budget: never spend the
538
+ // whole deadline waiting only to time out reading the cleared page.
539
+ const budget = Math.min(challengeWaitMs, deadline.remainingMs() - CHALLENGE_FINISH_RESERVE_MS)
540
+ if (budget <= 0) return false
541
+ const until = Date.now() + budget
542
+ for (;;) {
543
+ if (deadline.signal.aborted) throw new Error('challenge wait aborted')
544
+ if (await probeStillOnChallenge(page)) {
545
+ const remaining = until - Date.now()
546
+ if (remaining <= 0) return false
547
+ await sleep(Math.min(CHALLENGE_POLL_INTERVAL_MS, remaining))
548
+ continue
549
+ }
550
+ // Markers gone: let the fresh document reach domcontentloaded before
551
+ // the caller reads it (the reload may still be committing).
552
+ await page.waitForLoadState('domcontentloaded', { timeout: Math.min(deadline.remainingMs(), 2_000) }).catch(() => {})
553
+ return true
554
+ }
555
+ }
556
+ }
557
+
558
+ /** Resolve after `ms` — the bounded wait's inter-poll nap. */
559
+ function sleep(ms: number): Promise<void> {
560
+ return new Promise(resolve => { setTimeout(resolve, ms) })
561
+ }
562
+
563
+ /**
564
+ * Whether a response event belongs to the page's main frame as a document
565
+ * navigation — the filter the challenge wait uses to keep the LAST such
566
+ * response (challenge pages reload the same URL into the real document).
567
+ * Structural members are optional; when the backend does not expose them the
568
+ * check degrades to "looks like a document" so fakes stay usable.
569
+ */
570
+ function isMainFrameDocument(response: PlaywrightResponse, page: PlaywrightPage): boolean {
571
+ const request = response.request?.()
572
+ if (request === undefined) return true
573
+ if (typeof request.isNavigationRequest === 'function' && !request.isNavigationRequest()) return false
574
+ const resourceType = typeof request.resourceType === 'function' ? request.resourceType() : undefined
575
+ if (resourceType !== undefined && resourceType !== 'document') return false
576
+ if (typeof request.frame === 'function' && typeof page.mainFrame === 'function') {
577
+ return request.frame() === page.mainFrame()
578
+ }
579
+ return true
580
+ }
581
+
582
+ /** The running record of the last main-frame navigation response. */
583
+ interface MainFrameTracker {
584
+ /** The most recent main-frame navigation response seen, if any. */
585
+ last(): PlaywrightResponse | null
586
+ /** Pin the tracker to a goto's return (it is that navigation's response). */
587
+ seed(response: PlaywrightResponse | null): void
588
+ }
589
+
590
+ /**
591
+ * Watch the page for main-frame navigation responses so a challenge that
592
+ * reloads into the real document hands the caller THAT response — status,
593
+ * headers — instead of the challenge's. Best-effort: a page without the
594
+ * listener falls back to SPA-style content polling only.
595
+ */
596
+ function trackMainFrameResponses(page: PlaywrightPage): MainFrameTracker {
597
+ let last: PlaywrightResponse | null = null
598
+ try {
599
+ page.on?.('response', (response) => {
600
+ if (isMainFrameDocument(response, page)) last = response
601
+ })
602
+ } catch {
603
+ // keep going without response tracking
604
+ }
605
+ return {
606
+ last: () => last,
607
+ seed: (response) => { if (response !== null) last = response },
608
+ }
609
+ }
610
+
611
+ /**
612
+ * Probe the live document for challenge markers. Prefers the in-page probe
613
+ * (`page.evaluate`) — the only thing that can see SPA-style clears — and
614
+ * falls back to reading and classifying the serialized HTML when the backend
615
+ * does not expose evaluate. A probe that throws mid-navigation (execution
616
+ * context destroyed while the challenge reloads) counts as "still on the
617
+ * challenge": the next poll sees the fresh document.
618
+ */
619
+ async function probeStillOnChallenge(page: PlaywrightPage): Promise<boolean> {
620
+ if (typeof page.evaluate === 'function') {
621
+ try {
622
+ const verdict = await page.evaluate(CHALLENGE_DOM_PROBE)
623
+ if (typeof verdict === 'boolean') return verdict
624
+ } catch {
625
+ // fall through to the content check
626
+ }
627
+ }
628
+ try {
629
+ return classifyChallengeHtml(await page.content()) !== 'none'
630
+ } catch {
631
+ return true
632
+ }
420
633
  }
421
634
 
422
635
  /** Apply the decoded-body cap and flag the cut. */
package/src/types.ts CHANGED
@@ -13,6 +13,23 @@ export interface PlaywrightResponse {
13
13
  status(): number
14
14
  headers(): Record<string, string>
15
15
  text(): Promise<string>
16
+ /** This response's URL; absent on minimal fakes. */
17
+ url?(): string
18
+ /**
19
+ * The request this response answers — used to recognize main-frame
20
+ * navigation responses while the challenge wait runs. Absent on fakes.
21
+ */
22
+ request?(): PlaywrightRequest
23
+ }
24
+
25
+ /** The request side of a response, for main-frame filtering. */
26
+ export interface PlaywrightRequest {
27
+ /** True for navigations (document loads and their redirect hops). */
28
+ isNavigationRequest?(): boolean
29
+ /** `'document'` for frame navigations; absent on minimal fakes. */
30
+ resourceType?(): string
31
+ /** The frame that issued the request; compare with `page.mainFrame()`. */
32
+ frame?(): unknown
16
33
  }
17
34
 
18
35
  /** A page inside a context. */
@@ -29,6 +46,19 @@ export interface PlaywrightPage {
29
46
  route(glob: string, handler: (route: PlaywrightRoute) => Promise<void>): Promise<void>
30
47
  /** Popup notification; the fetch closes whatever its page spawns. */
31
48
  on?(event: 'popup', listener: (page: PlaywrightPage) => void): unknown
49
+ /**
50
+ * Response notification — the challenge wait uses it to track the LAST
51
+ * main-frame navigation response (challenge pages reload into the real
52
+ * document). Absent on minimal fakes (content polling covers them).
53
+ */
54
+ on?(event: 'response', listener: (response: PlaywrightResponse) => void): unknown
55
+ /**
56
+ * Evaluate an expression in the page — the challenge probe's live-DOM
57
+ * path. Absent on minimal fakes (content polling covers them).
58
+ */
59
+ evaluate?(script: string, arg?: unknown): Promise<unknown>
60
+ /** The main frame handle; compare with `request.frame()` for filtering. */
61
+ mainFrame?(): unknown
32
62
  }
33
63
 
34
64
  /**