dsh-web-fetch-playwright 0.2.3 → 0.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -2
- package/README.zh-CN.md +18 -2
- package/dsh.plugin.json +1 -1
- package/lib/client.js +52 -27
- package/lib/index.js +451 -15
- package/lib/types/challenge.d.ts +83 -0
- package/lib/types/client/controller.d.ts +7 -0
- package/lib/types/client/locales.d.ts +1 -1
- package/lib/types/config.d.ts +42 -0
- package/lib/types/index.d.ts +4 -2
- package/lib/types/markdown.d.ts +2 -0
- package/lib/types/provider.d.ts +44 -1
- package/lib/types/types.d.ts +29 -0
- package/package.json +9 -7
- package/src/challenge.ts +216 -0
- package/src/client/card.tsx +13 -0
- package/src/client/controller.ts +9 -0
- package/src/client/locales.ts +7 -0
- package/src/config.ts +57 -0
- package/src/index.ts +4 -2
- package/src/markdown.ts +55 -0
- package/src/provider.ts +223 -10
- package/src/types.ts +30 -0
package/src/markdown.ts
CHANGED
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
* The denoise pipeline: rendered HTML → sanitized article HTML → markdown.
|
|
3
3
|
*
|
|
4
4
|
* The classic stack, in order: jsdom parses the page Playwright rendered;
|
|
5
|
+
* inline `data:` image payloads are elided to size placeholders (build
|
|
6
|
+
* tools inline images as base64, which would otherwise dominate the body);
|
|
5
7
|
* Mozilla Readability extracts the article (dropping nav bars, sidebars,
|
|
6
8
|
* footers, and ad chrome by scoring link density and text mass); DOMPurify
|
|
7
9
|
* sanitizes whatever HTML remains and forbids the layout tags noise lives in;
|
|
@@ -30,6 +32,52 @@ const FORBID_TAGS = [
|
|
|
30
32
|
/** Attributes stripped from sanitized output (styling survives as noise). */
|
|
31
33
|
const FORBID_ATTR = ['style', 'class', 'id', 'hidden', 'aria-hidden', 'role']
|
|
32
34
|
|
|
35
|
+
/**
|
|
36
|
+
* Elide inline `data:` image payloads to `data:<mime>;base64,...<size>`.
|
|
37
|
+
*
|
|
38
|
+
* Build tools (Docusaurus/webpack, Hugo) inline images above a size cutoff
|
|
39
|
+
* straight into the HTML as data URIs — measured on an onlyoffice.com docs
|
|
40
|
+
* page, 12 inline PNGs were 21.6% of the HTML and, surviving Readability
|
|
41
|
+
* (they are content), DOMPurify (img+data: is on its DATA_URI_TAGS
|
|
42
|
+
* allowlist), and Turndown's default image rule, ended up **65% of the
|
|
43
|
+
* returned markdown body**. Network-level filtering cannot touch them: a
|
|
44
|
+
* data URI is never fetched, so the provider's subrequest abort misses it.
|
|
45
|
+
* The placeholder keeps alt text, MIME type, and the approximate size, so
|
|
46
|
+
* the model still knows an inline image existed. The marker is pure ASCII
|
|
47
|
+
* because Readability re-resolves image srcs through `new URL()`, which
|
|
48
|
+
* would percent-encode anything else.
|
|
49
|
+
*/
|
|
50
|
+
function elideDataUriImages(document: Document): void {
|
|
51
|
+
for (const img of Array.from(document.querySelectorAll('img'))) {
|
|
52
|
+
const src = img.getAttribute('src')
|
|
53
|
+
if (src === null || !src.startsWith('data:')) continue
|
|
54
|
+
img.setAttribute('src', dataUriPlaceholder(src))
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Shorten one data URI to its header plus a size marker.
|
|
60
|
+
* @param src - the original `data:` URI.
|
|
61
|
+
* @returns e.g. `data:image/png;base64,...8.9KB` (base64 sizes are decoded
|
|
62
|
+
* bytes; non-base64 payloads report their character count).
|
|
63
|
+
*/
|
|
64
|
+
function dataUriPlaceholder(src: string): string {
|
|
65
|
+
const commaIndex = src.indexOf(',')
|
|
66
|
+
const header = commaIndex === -1 ? src : src.slice(0, commaIndex + 1)
|
|
67
|
+
const payload = commaIndex === -1 ? '' : src.slice(commaIndex + 1)
|
|
68
|
+
const size = /;base64$/i.test(header.slice(5, -1))
|
|
69
|
+
? Math.max(0, Math.floor(payload.length / 4) * 3 - (payload.match(/=+$/)?.[0].length ?? 0))
|
|
70
|
+
: payload.length
|
|
71
|
+
return `${header}...${humanSize(size)}`
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** Render a byte count as `123B` / `8.9KB` / `1.2MB`. */
|
|
75
|
+
function humanSize(bytes: number): string {
|
|
76
|
+
if (bytes < 1024) return `${String(bytes)}B`
|
|
77
|
+
if (bytes < 1_048_576) return `${(bytes / 1024).toFixed(1)}KB`
|
|
78
|
+
return `${(bytes / 1_048_576).toFixed(1)}MB`
|
|
79
|
+
}
|
|
80
|
+
|
|
33
81
|
/** The shared converter: same style options as `dsh-tool-web`'s renderer. */
|
|
34
82
|
const turndown = new TurndownService({
|
|
35
83
|
headingStyle: 'atx',
|
|
@@ -115,6 +163,13 @@ export function htmlToMarkdown(html: string, url: string): DenoiseResult {
|
|
|
115
163
|
const purify = createDOMPurify(dom.window)
|
|
116
164
|
const document = dom.window.document
|
|
117
165
|
|
|
166
|
+
// Before anything downstream reads the DOM: shrink inline data-URI image
|
|
167
|
+
// payloads to size placeholders. Mutating the live document here covers
|
|
168
|
+
// both paths — the Readability clone below, and the document.body fallback
|
|
169
|
+
// (innerHTML reflects the rewritten src) — while Turndown's default image
|
|
170
|
+
// rule keeps handling alt/title escaping on the already-short src.
|
|
171
|
+
elideDataUriImages(document)
|
|
172
|
+
|
|
118
173
|
let source: string | null = null
|
|
119
174
|
let title: string | undefined
|
|
120
175
|
try {
|
package/src/provider.ts
CHANGED
|
@@ -22,21 +22,44 @@
|
|
|
22
22
|
* browser can reach. Profile mode additionally acts WITH the remote
|
|
23
23
|
* browser's logged-in sessions (see the README's risk notes).
|
|
24
24
|
*
|
|
25
|
+
* Cloudflare challenges (issue #2): when a navigation lands on a challenge
|
|
26
|
+
* interstitial, the fetch waits — on the SAME page and in the SAME browser
|
|
27
|
+
* context, so the browser's natural verification and any clearance cookies
|
|
28
|
+
* it earns apply — for a bounded, configurable window
|
|
29
|
+
* (`challengeWaitMs`, default 15s; 0 restores the legacy first-response
|
|
30
|
+
* behavior). The wait tracks the LAST main-frame navigation response (the
|
|
31
|
+
* real page reloads in after the challenge clears) and watches the live DOM
|
|
32
|
+
* so SPA-style clears are caught too. It never clicks, never injects
|
|
33
|
+
* answers, never touches cookies itself; when the budget runs out the fetch
|
|
34
|
+
* fails with {@link WEB_FETCH_CHALLENGE_CODE} instead of returning the
|
|
35
|
+
* interstitial as content.
|
|
36
|
+
*
|
|
25
37
|
* @module dsh-web-fetch-playwright/provider
|
|
26
38
|
*/
|
|
27
39
|
|
|
28
40
|
import { WebError } from '@deepseek-ai/dsh-web'
|
|
29
41
|
import type { WebFetchProvider, WebFetchRequest, WebFetchResult } from '@deepseek-ai/dsh-web'
|
|
30
|
-
import {
|
|
42
|
+
import { CHALLENGE_DOM_PROBE, CHALLENGE_FINISH_RESERVE_MS, CHALLENGE_POLL_INTERVAL_MS, classifyChallengeHtml, classifyChallengeResponse, isChallengeCompatibleResponse } from './challenge.ts'
|
|
43
|
+
import type { ChallengeVerdict } from './challenge.ts'
|
|
44
|
+
import { DEFAULT_MAX_CONCURRENCY_CDP, DEFAULT_MAX_CONCURRENCY_LOCAL, effectiveChallengeRetries, effectiveChallengeWaitMs, effectiveContextMode, effectiveMaxConcurrency, normalizeCdpEndpoint } from './config.ts'
|
|
31
45
|
import type { ResolvedConfig } from './config.ts'
|
|
32
46
|
import { CdpConnectionPool } from './cdp-pool.ts'
|
|
33
47
|
import { htmlToMarkdown } from './markdown.ts'
|
|
34
48
|
import { resolveCdpBackend, resolvePlaywrightBackend } from './playwright-resolve.ts'
|
|
35
|
-
import type { PlaywrightBrowser, PlaywrightContext, PlaywrightPage, PlaywrightRoute } from './types.ts'
|
|
49
|
+
import type { PlaywrightBrowser, PlaywrightContext, PlaywrightPage, PlaywrightResponse, PlaywrightRoute } from './types.ts'
|
|
36
50
|
|
|
37
51
|
/** Stable id this provider registers under (the bundle patch pins it). */
|
|
38
52
|
export const PLAYWRIGHT_FETCH_PROVIDER_ID = 'playwright'
|
|
39
53
|
|
|
54
|
+
/**
|
|
55
|
+
* Error code for a Cloudflare challenge the bounded natural wait could not
|
|
56
|
+
* clear. Provider-specific by design — the web seam's `code` is an open
|
|
57
|
+
* string and consumers must tolerate provider-specific codes — so callers
|
|
58
|
+
* can tell "the site challenged us and the browser did not pass" apart from
|
|
59
|
+
* a transport timeout or a provider bug.
|
|
60
|
+
*/
|
|
61
|
+
export const WEB_FETCH_CHALLENGE_CODE = 'WEB_FETCH_CHALLENGE'
|
|
62
|
+
|
|
40
63
|
/** Maximum accepted request URL length (http-provider parity). */
|
|
41
64
|
const MAX_URL_LENGTH = 2048
|
|
42
65
|
|
|
@@ -378,7 +401,11 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
378
401
|
}
|
|
379
402
|
}
|
|
380
403
|
|
|
381
|
-
/**
|
|
404
|
+
/**
|
|
405
|
+
* Navigate, settle, and decode one URL inside an open session's tab —
|
|
406
|
+
* waiting out any Cloudflare interstitial within the bounded challenge
|
|
407
|
+
* budget before the final document is read (see the class docs).
|
|
408
|
+
*/
|
|
382
409
|
private async retrieve(
|
|
383
410
|
session: BrowserSession,
|
|
384
411
|
url: URL,
|
|
@@ -386,18 +413,70 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
386
413
|
deadline: Deadline,
|
|
387
414
|
): Promise<WebFetchResult> {
|
|
388
415
|
const page = session.page
|
|
389
|
-
const
|
|
416
|
+
const challengeWaitMs = effectiveChallengeWaitMs(config)
|
|
417
|
+
// Feature switch: 0 keeps the exact legacy (pre-0.2.5) behavior — the
|
|
418
|
+
// first response decides, no waiting — an escape hatch and the A/B
|
|
419
|
+
// baseline every test proves the bug against.
|
|
420
|
+
const tracker = challengeWaitMs > 0 ? trackMainFrameResponses(page) : undefined
|
|
421
|
+
let response = await page.goto(url.toString(), { waitUntil: 'domcontentloaded', timeout: deadline.remainingMs() })
|
|
422
|
+
tracker?.seed(response)
|
|
423
|
+
|
|
424
|
+
let challengeEntryResponse: PlaywrightResponse | null = null
|
|
425
|
+
if (challengeWaitMs > 0) {
|
|
426
|
+
let attemptsLeft = effectiveChallengeRetries(config) + 1
|
|
427
|
+
for (;;) {
|
|
428
|
+
const verdict = await this.verdictAfterLoad(page, tracker?.last() ?? response)
|
|
429
|
+
if (verdict === 'blocked') {
|
|
430
|
+
throw new WebError(
|
|
431
|
+
`the site hard-blocked this fetch at its Cloudflare edge (waiting cannot clear it): ${page.url()}`,
|
|
432
|
+
WEB_FETCH_CHALLENGE_CODE,
|
|
433
|
+
)
|
|
434
|
+
}
|
|
435
|
+
if (verdict !== 'challenge') break
|
|
436
|
+
challengeEntryResponse = tracker?.last() ?? response
|
|
437
|
+
const cleared = await this.waitForChallengeClear(page, deadline, challengeWaitMs)
|
|
438
|
+
if (cleared) {
|
|
439
|
+
// The settled document may be the next round of a CHAINED
|
|
440
|
+
// challenge (JS test → Turnstile interstitial): the probe can
|
|
441
|
+
// clear in the gap between rounds, so confirm on the settled
|
|
442
|
+
// DOM. Deliberately content-level — an SPA clear keeps the 403
|
|
443
|
+
// challenge response forever and must still pass here.
|
|
444
|
+
let chained = false
|
|
445
|
+
try { chained = classifyChallengeHtml(await page.content()) === 'challenge' } catch { chained = false }
|
|
446
|
+
if (!chained) break
|
|
447
|
+
}
|
|
448
|
+
if (--attemptsLeft <= 0) {
|
|
449
|
+
const lastStatus = (tracker?.last() ?? response)?.status()
|
|
450
|
+
throw new WebError(
|
|
451
|
+
`the site kept serving a Cloudflare challenge (last status ${lastStatus === undefined ? 'unknown' : String(lastStatus)}) for up to ${String(challengeWaitMs)}ms across ${String(effectiveChallengeRetries(config) + 1)} attempt(s); the browser did not clear it naturally — retry later, raise challengeWaitMs, or use a profile whose browser already holds clearance`,
|
|
452
|
+
WEB_FETCH_CHALLENGE_CODE,
|
|
453
|
+
)
|
|
454
|
+
}
|
|
455
|
+
// Same page, same context — after an expired window OR a chained
|
|
456
|
+
// round: any clearance cookies already earned stay in the jar for
|
|
457
|
+
// this one retry, then everything is torn down as usual.
|
|
458
|
+
response = await page.goto(url.toString(), { waitUntil: 'domcontentloaded', timeout: deadline.remainingMs() })
|
|
459
|
+
tracker?.seed(response)
|
|
460
|
+
}
|
|
461
|
+
}
|
|
390
462
|
|
|
391
|
-
|
|
463
|
+
// Final document: the LAST main-frame response when the challenge wait
|
|
464
|
+
// ran (the real page reloads in), else the response goto returned.
|
|
465
|
+
const finalResponse = (challengeWaitMs > 0 ? tracker?.last() : undefined) ?? response
|
|
466
|
+
const kind = classifyContentType(finalResponse?.headers()['content-type'])
|
|
392
467
|
if (kind === undefined) {
|
|
393
|
-
throw new WebError(`unsupported content type "${
|
|
468
|
+
throw new WebError(`unsupported content type "${finalResponse?.headers()['content-type'] ?? 'unknown'}"`, 'WEB_UNSUPPORTED_CONTENT_TYPE')
|
|
394
469
|
}
|
|
395
470
|
const finalUrl = page.url()
|
|
471
|
+
// An SPA-style clear swaps the document without navigating: no new
|
|
472
|
+
// response exists to report, so the cleared document reads as served.
|
|
473
|
+
const clearedWithoutNavigation = challengeEntryResponse !== null && finalResponse === challengeEntryResponse
|
|
474
|
+
const statusCode = finalResponse !== null && !clearedWithoutNavigation ? finalResponse.status() : 200
|
|
396
475
|
|
|
397
476
|
// Non-HTML decodes straight from the response body; no denoise applies.
|
|
398
477
|
if (kind === 'text') {
|
|
399
|
-
const text =
|
|
400
|
-
return capResult(finalUrl,
|
|
478
|
+
const text = finalResponse !== null ? await finalResponse.text() : await page.content()
|
|
479
|
+
return capResult(finalUrl, statusCode, { kind: 'text', content: text })
|
|
401
480
|
}
|
|
402
481
|
|
|
403
482
|
// Best-effort settle for client-rendered content; a timeout just keeps
|
|
@@ -408,15 +487,149 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
408
487
|
if (!config.denoise) {
|
|
409
488
|
// The tool layer's own turndown renders raw HTML; the checkbox only
|
|
410
489
|
// governs the Readability/DOMPurify stage this provider owns.
|
|
411
|
-
return capResult(finalUrl,
|
|
490
|
+
return capResult(finalUrl, statusCode, { kind: 'html', content: html })
|
|
412
491
|
}
|
|
413
492
|
const bounded = html.length > MAX_PIPELINE_INPUT_CHARS ? html.slice(0, MAX_PIPELINE_INPUT_CHARS) : html
|
|
414
493
|
const { markdown } = htmlToMarkdown(bounded, finalUrl)
|
|
415
|
-
const result = capResult(finalUrl,
|
|
494
|
+
const result = capResult(finalUrl, statusCode, { kind: 'text', content: markdown })
|
|
416
495
|
return bounded !== html
|
|
417
496
|
? { ...result, truncated: true }
|
|
418
497
|
: result
|
|
419
498
|
}
|
|
499
|
+
|
|
500
|
+
/**
|
|
501
|
+
* Whether the freshly loaded document is a Cloudflare interstitial: the
|
|
502
|
+
* response-level signals first (the documented `cf-mitigated` header, then
|
|
503
|
+
* the 403/503 + cloudflare fallback), then — gated behind
|
|
504
|
+
* {@link isChallengeCompatibleResponse}, because interstitials never ship
|
|
505
|
+
* a plain 200 — the localized title family and structural markers, which
|
|
506
|
+
* also recognize the hard-block page waiting can never clear.
|
|
507
|
+
*/
|
|
508
|
+
private async verdictAfterLoad(page: PlaywrightPage, response: PlaywrightResponse | null): Promise<ChallengeVerdict> {
|
|
509
|
+
if (response !== null && classifyChallengeResponse(response.status(), response.headers()) === 'challenge') return 'challenge'
|
|
510
|
+
// Suspicion gate: the content-level fallback only runs on a
|
|
511
|
+
// challenge-compatible response, so a normal article cannot be misread
|
|
512
|
+
// as a challenge no matter what it quotes. A response-less navigation
|
|
513
|
+
// keeps the check — its headers are simply unavailable.
|
|
514
|
+
if (response !== null && !isChallengeCompatibleResponse(response.status(), response.headers())) return 'none'
|
|
515
|
+
// Only HTML documents can be interstitials; classifyContentType already
|
|
516
|
+
// reads a missing content type as html (a response-less navigation).
|
|
517
|
+
if (response === null || classifyContentType(response.headers()['content-type']) === 'html') {
|
|
518
|
+
try {
|
|
519
|
+
return classifyChallengeHtml(await page.content())
|
|
520
|
+
} catch {
|
|
521
|
+
return 'none' // a document we cannot read is not a challenge we can wait on
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
return 'none'
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
/**
|
|
528
|
+
* The bounded natural wait: poll the live document for challenge markers
|
|
529
|
+
* until they are gone (the user's real browser passed the verification and
|
|
530
|
+
* reloaded into the real page, or swapped it in SPA-style), the per-fetch
|
|
531
|
+
* wait budget runs out, or the fetch's deadline aborts. No clicks, no
|
|
532
|
+
* injected answers — the browser either clears it on its own or it does not.
|
|
533
|
+
*
|
|
534
|
+
* @returns true when the challenge cleared; false when the budget ran out.
|
|
535
|
+
*/
|
|
536
|
+
private async waitForChallengeClear(page: PlaywrightPage, deadline: Deadline, challengeWaitMs: number): Promise<boolean> {
|
|
537
|
+
// Keep the settle + decode tail inside the fetch budget: never spend the
|
|
538
|
+
// whole deadline waiting only to time out reading the cleared page.
|
|
539
|
+
const budget = Math.min(challengeWaitMs, deadline.remainingMs() - CHALLENGE_FINISH_RESERVE_MS)
|
|
540
|
+
if (budget <= 0) return false
|
|
541
|
+
const until = Date.now() + budget
|
|
542
|
+
for (;;) {
|
|
543
|
+
if (deadline.signal.aborted) throw new Error('challenge wait aborted')
|
|
544
|
+
if (await probeStillOnChallenge(page)) {
|
|
545
|
+
const remaining = until - Date.now()
|
|
546
|
+
if (remaining <= 0) return false
|
|
547
|
+
await sleep(Math.min(CHALLENGE_POLL_INTERVAL_MS, remaining))
|
|
548
|
+
continue
|
|
549
|
+
}
|
|
550
|
+
// Markers gone: let the fresh document reach domcontentloaded before
|
|
551
|
+
// the caller reads it (the reload may still be committing).
|
|
552
|
+
await page.waitForLoadState('domcontentloaded', { timeout: Math.min(deadline.remainingMs(), 2_000) }).catch(() => {})
|
|
553
|
+
return true
|
|
554
|
+
}
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
/** Resolve after `ms` — the bounded wait's inter-poll nap. */
|
|
559
|
+
function sleep(ms: number): Promise<void> {
|
|
560
|
+
return new Promise(resolve => { setTimeout(resolve, ms) })
|
|
561
|
+
}
|
|
562
|
+
|
|
563
|
+
/**
|
|
564
|
+
* Whether a response event belongs to the page's main frame as a document
|
|
565
|
+
* navigation — the filter the challenge wait uses to keep the LAST such
|
|
566
|
+
* response (challenge pages reload the same URL into the real document).
|
|
567
|
+
* Structural members are optional; when the backend does not expose them the
|
|
568
|
+
* check degrades to "looks like a document" so fakes stay usable.
|
|
569
|
+
*/
|
|
570
|
+
function isMainFrameDocument(response: PlaywrightResponse, page: PlaywrightPage): boolean {
|
|
571
|
+
const request = response.request?.()
|
|
572
|
+
if (request === undefined) return true
|
|
573
|
+
if (typeof request.isNavigationRequest === 'function' && !request.isNavigationRequest()) return false
|
|
574
|
+
const resourceType = typeof request.resourceType === 'function' ? request.resourceType() : undefined
|
|
575
|
+
if (resourceType !== undefined && resourceType !== 'document') return false
|
|
576
|
+
if (typeof request.frame === 'function' && typeof page.mainFrame === 'function') {
|
|
577
|
+
return request.frame() === page.mainFrame()
|
|
578
|
+
}
|
|
579
|
+
return true
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
/** The running record of the last main-frame navigation response. */
|
|
583
|
+
interface MainFrameTracker {
|
|
584
|
+
/** The most recent main-frame navigation response seen, if any. */
|
|
585
|
+
last(): PlaywrightResponse | null
|
|
586
|
+
/** Pin the tracker to a goto's return (it is that navigation's response). */
|
|
587
|
+
seed(response: PlaywrightResponse | null): void
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
/**
|
|
591
|
+
* Watch the page for main-frame navigation responses so a challenge that
|
|
592
|
+
* reloads into the real document hands the caller THAT response — status,
|
|
593
|
+
* headers — instead of the challenge's. Best-effort: a page without the
|
|
594
|
+
* listener falls back to SPA-style content polling only.
|
|
595
|
+
*/
|
|
596
|
+
function trackMainFrameResponses(page: PlaywrightPage): MainFrameTracker {
|
|
597
|
+
let last: PlaywrightResponse | null = null
|
|
598
|
+
try {
|
|
599
|
+
page.on?.('response', (response) => {
|
|
600
|
+
if (isMainFrameDocument(response, page)) last = response
|
|
601
|
+
})
|
|
602
|
+
} catch {
|
|
603
|
+
// keep going without response tracking
|
|
604
|
+
}
|
|
605
|
+
return {
|
|
606
|
+
last: () => last,
|
|
607
|
+
seed: (response) => { if (response !== null) last = response },
|
|
608
|
+
}
|
|
609
|
+
}
|
|
610
|
+
|
|
611
|
+
/**
|
|
612
|
+
* Probe the live document for challenge markers. Prefers the in-page probe
|
|
613
|
+
* (`page.evaluate`) — the only thing that can see SPA-style clears — and
|
|
614
|
+
* falls back to reading and classifying the serialized HTML when the backend
|
|
615
|
+
* does not expose evaluate. A probe that throws mid-navigation (execution
|
|
616
|
+
* context destroyed while the challenge reloads) counts as "still on the
|
|
617
|
+
* challenge": the next poll sees the fresh document.
|
|
618
|
+
*/
|
|
619
|
+
async function probeStillOnChallenge(page: PlaywrightPage): Promise<boolean> {
|
|
620
|
+
if (typeof page.evaluate === 'function') {
|
|
621
|
+
try {
|
|
622
|
+
const verdict = await page.evaluate(CHALLENGE_DOM_PROBE)
|
|
623
|
+
if (typeof verdict === 'boolean') return verdict
|
|
624
|
+
} catch {
|
|
625
|
+
// fall through to the content check
|
|
626
|
+
}
|
|
627
|
+
}
|
|
628
|
+
try {
|
|
629
|
+
return classifyChallengeHtml(await page.content()) !== 'none'
|
|
630
|
+
} catch {
|
|
631
|
+
return true
|
|
632
|
+
}
|
|
420
633
|
}
|
|
421
634
|
|
|
422
635
|
/** Apply the decoded-body cap and flag the cut. */
|
package/src/types.ts
CHANGED
|
@@ -13,6 +13,23 @@ export interface PlaywrightResponse {
|
|
|
13
13
|
status(): number
|
|
14
14
|
headers(): Record<string, string>
|
|
15
15
|
text(): Promise<string>
|
|
16
|
+
/** This response's URL; absent on minimal fakes. */
|
|
17
|
+
url?(): string
|
|
18
|
+
/**
|
|
19
|
+
* The request this response answers — used to recognize main-frame
|
|
20
|
+
* navigation responses while the challenge wait runs. Absent on fakes.
|
|
21
|
+
*/
|
|
22
|
+
request?(): PlaywrightRequest
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/** The request side of a response, for main-frame filtering. */
|
|
26
|
+
export interface PlaywrightRequest {
|
|
27
|
+
/** True for navigations (document loads and their redirect hops). */
|
|
28
|
+
isNavigationRequest?(): boolean
|
|
29
|
+
/** `'document'` for frame navigations; absent on minimal fakes. */
|
|
30
|
+
resourceType?(): string
|
|
31
|
+
/** The frame that issued the request; compare with `page.mainFrame()`. */
|
|
32
|
+
frame?(): unknown
|
|
16
33
|
}
|
|
17
34
|
|
|
18
35
|
/** A page inside a context. */
|
|
@@ -29,6 +46,19 @@ export interface PlaywrightPage {
|
|
|
29
46
|
route(glob: string, handler: (route: PlaywrightRoute) => Promise<void>): Promise<void>
|
|
30
47
|
/** Popup notification; the fetch closes whatever its page spawns. */
|
|
31
48
|
on?(event: 'popup', listener: (page: PlaywrightPage) => void): unknown
|
|
49
|
+
/**
|
|
50
|
+
* Response notification — the challenge wait uses it to track the LAST
|
|
51
|
+
* main-frame navigation response (challenge pages reload into the real
|
|
52
|
+
* document). Absent on minimal fakes (content polling covers them).
|
|
53
|
+
*/
|
|
54
|
+
on?(event: 'response', listener: (response: PlaywrightResponse) => void): unknown
|
|
55
|
+
/**
|
|
56
|
+
* Evaluate an expression in the page — the challenge probe's live-DOM
|
|
57
|
+
* path. Absent on minimal fakes (content polling covers them).
|
|
58
|
+
*/
|
|
59
|
+
evaluate?(script: string, arg?: unknown): Promise<unknown>
|
|
60
|
+
/** The main frame handle; compare with `request.frame()` for filtering. */
|
|
61
|
+
mainFrame?(): unknown
|
|
32
62
|
}
|
|
33
63
|
|
|
34
64
|
/**
|