dsh-web-fetch-playwright 0.2.3 → 0.2.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -24,6 +24,19 @@ export declare const DEFAULT_MAX_CONCURRENCY_LOCAL = 4;
24
24
  export declare const DEFAULT_MAX_CONCURRENCY_CDP = 50;
25
25
  /** Ceiling the schema accepts for `maxConcurrency` (local slots are browsers). */
26
26
  export declare const MAX_CONCURRENCY_CEILING = 200;
27
+ /**
28
+ * Default bounded wait (ms) for a Cloudflare challenge to clear naturally —
29
+ * the user's real browser passes the verification on its own while the fetch
30
+ * holds the same page and context. Sized to fit the 45s per-fetch deadline
31
+ * with the settle/decode tail (and one retry) still inside it.
32
+ */
33
+ export declare const DEFAULT_CHALLENGE_WAIT_MS = 15000;
34
+ /** Ceiling the schema accepts for `challengeWaitMs`. */
35
+ export declare const MAX_CHALLENGE_WAIT_MS = 60000;
36
+ /** Default same-page re-navigation attempts after a challenge wait runs out. */
37
+ export declare const DEFAULT_CHALLENGE_RETRIES = 1;
38
+ /** Ceiling the schema accepts for `challengeRetries`. */
39
+ export declare const MAX_CHALLENGE_RETRIES = 3;
27
40
  /** Which browser backend serves a fetch. */
28
41
  export type PlaywrightBackend = 'local' | 'cdp';
29
42
  /**
@@ -53,6 +66,19 @@ export interface Config {
53
66
  shareBrowserContext?: boolean;
54
67
  /** Whether the Readability + DOMPurify denoise pipeline runs before markdown. */
55
68
  denoise?: boolean;
69
+ /**
70
+ * Bounded wait (ms) for a Cloudflare challenge to clear naturally inside
71
+ * the same page/context. `0` (or any config leaving this at 0) restores the
72
+ * legacy behavior: the first response is final, no waiting — the A/B
73
+ * baseline. Schema default: {@link DEFAULT_CHALLENGE_WAIT_MS}.
74
+ */
75
+ challengeWaitMs?: number;
76
+ /**
77
+ * Same-page re-navigation attempts after a challenge wait window runs out
78
+ * (any partial clearance cookies stay in the context for the retry).
79
+ * Schema default: {@link DEFAULT_CHALLENGE_RETRIES}.
80
+ */
81
+ challengeRetries?: number;
56
82
  /**
57
83
  * How many fetches may render at once. Blank = backend default (4 for
58
84
  * local — each slot launches a browser; 50 for CDP — each slot is a tab in
@@ -78,6 +104,22 @@ export type ResolvedConfig = Omit<Required<Config>, 'maxConcurrency'> & {
78
104
  * @returns the effective limit for the semaphore.
79
105
  */
80
106
  export declare function effectiveMaxConcurrency(config: Pick<Config, 'backend' | 'maxConcurrency'>): number;
107
+ /**
108
+ * The challenge wait budget a fetch actually runs with: an explicit setting
109
+ * wins, else the schema default. `0` means the challenge path is off.
110
+ *
111
+ * @param config - the resolved settings section (or any partial of it).
112
+ * @returns the bounded wait in milliseconds.
113
+ */
114
+ export declare function effectiveChallengeWaitMs(config: Pick<Config, 'challengeWaitMs'>): number;
115
+ /**
116
+ * The same-page retry count a challenge fetch actually runs with: an explicit
117
+ * setting wins, else the schema default.
118
+ *
119
+ * @param config - the resolved settings section (or any partial of it).
120
+ * @returns retries after a wait window runs out.
121
+ */
122
+ export declare function effectiveChallengeRetries(config: Pick<Config, 'challengeRetries'>): number;
81
123
  /**
82
124
  * The context mode a fetch actually runs with: profile (a tab in the remote
83
125
  * browser's real profile) requires the CDP backend — a local launch has no
@@ -11,11 +11,13 @@
11
11
  */
12
12
  import type { Context } from '@deepseek-ai/cordis';
13
13
  import { Config } from './config.ts';
14
- export { Config, DEFAULT_CDP_ENDPOINT, effectiveContextMode, normalizeCdpEndpoint } from './config.ts';
14
+ export { Config, DEFAULT_CDP_ENDPOINT, DEFAULT_CHALLENGE_RETRIES, DEFAULT_CHALLENGE_WAIT_MS, effectiveChallengeRetries, effectiveChallengeWaitMs, effectiveContextMode, normalizeCdpEndpoint } from './config.ts';
15
15
  export type { Config as PlaywrightFetchConfig, CdpContextMode, PlaywrightBackend, ResolvedConfig } from './config.ts';
16
16
  export { CdpConnectionPool } from './cdp-pool.ts';
17
17
  export type { CdpAcquireMode, CdpConnect, CdpLease } from './cdp-pool.ts';
18
- export { PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider } from './provider.ts';
18
+ export { CHALLENGE_DOM_PROBE, CHALLENGE_FINISH_RESERVE_MS, CHALLENGE_POLL_INTERVAL_MS, CHALLENGE_TITLE_RE, classifyChallengeHtml, classifyChallengeResponse, isChallengeCompatibleResponse } from './challenge.ts';
19
+ export type { ChallengeVerdict } from './challenge.ts';
20
+ export { PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider, WEB_FETCH_CHALLENGE_CODE } from './provider.ts';
19
21
  export { htmlToMarkdown } from './markdown.ts';
20
22
  export type { DenoiseMode, DenoiseResult } from './markdown.ts';
21
23
  /** Cordis plugin name used by loader diagnostics. */
@@ -2,6 +2,8 @@
2
2
  * The denoise pipeline: rendered HTML → sanitized article HTML → markdown.
3
3
  *
4
4
  * The classic stack, in order: jsdom parses the page Playwright rendered;
5
+ * inline `data:` image payloads are elided to size placeholders (build
6
+ * tools inline images as base64, which would otherwise dominate the body);
5
7
  * Mozilla Readability extracts the article (dropping nav bars, sidebars,
6
8
  * footers, and ad chrome by scoring link density and text mass); DOMPurify
7
9
  * sanitizes whatever HTML remains and forbids the layout tags noise lives in;
@@ -22,6 +22,18 @@
22
22
  * browser can reach. Profile mode additionally acts WITH the remote
23
23
  * browser's logged-in sessions (see the README's risk notes).
24
24
  *
25
+ * Cloudflare challenges (issue #2): when a navigation lands on a challenge
26
+ * interstitial, the fetch waits — on the SAME page and in the SAME browser
27
+ * context, so the browser's natural verification and any clearance cookies
28
+ * it earns apply — for a bounded, configurable window
29
+ * (`challengeWaitMs`, default 15s; 0 restores the legacy first-response
30
+ * behavior). The wait tracks the LAST main-frame navigation response (the
31
+ * real page reloads in after the challenge clears) and watches the live DOM
32
+ * so SPA-style clears are caught too. It never clicks, never injects
33
+ * answers, never touches cookies itself; when the budget runs out the fetch
34
+ * fails with {@link WEB_FETCH_CHALLENGE_CODE} instead of returning the
35
+ * interstitial as content.
36
+ *
25
37
  * @module dsh-web-fetch-playwright/provider
26
38
  */
27
39
  import type { WebFetchProvider, WebFetchRequest, WebFetchResult } from '@deepseek-ai/dsh-web';
@@ -30,6 +42,14 @@ import { CdpConnectionPool } from './cdp-pool.ts';
30
42
  import type { PlaywrightBrowser, PlaywrightContext, PlaywrightPage } from './types.ts';
31
43
  /** Stable id this provider registers under (the bundle patch pins it). */
32
44
  export declare const PLAYWRIGHT_FETCH_PROVIDER_ID = "playwright";
45
+ /**
46
+ * Error code for a Cloudflare challenge the bounded natural wait could not
47
+ * clear. Provider-specific by design — the web seam's `code` is an open
48
+ * string and consumers must tolerate provider-specific codes — so callers
49
+ * can tell "the site challenged us and the browser did not pass" apart from
50
+ * a transport timeout or a provider bug.
51
+ */
52
+ export declare const WEB_FETCH_CHALLENGE_CODE = "WEB_FETCH_CHALLENGE";
33
53
  /**
34
54
  * One render session: a browser, the context this fetch works in, and the
35
55
  * tab it owns. Local backends own the browser too; CDP sessions ride the
@@ -95,7 +115,30 @@ export declare class PlaywrightFetchProvider implements WebFetchProvider {
95
115
  * @returns the browser session the fetch will use.
96
116
  */
97
117
  protected openSession(config: ResolvedConfig, deadline: Deadline): Promise<BrowserSession>;
98
- /** Navigate, settle, and decode one URL inside an open session's tab. */
118
+ /**
119
+ * Navigate, settle, and decode one URL inside an open session's tab —
120
+ * waiting out any Cloudflare interstitial within the bounded challenge
121
+ * budget before the final document is read (see the class docs).
122
+ */
99
123
  private retrieve;
124
+ /**
125
+ * Whether the freshly loaded document is a Cloudflare interstitial: the
126
+ * response-level signals first (the documented `cf-mitigated` header, then
127
+ * the 403/503 + cloudflare fallback), then — gated behind
128
+ * {@link isChallengeCompatibleResponse}, because interstitials never ship
129
+ * a plain 200 — the localized title family and structural markers, which
130
+ * also recognize the hard-block page waiting can never clear.
131
+ */
132
+ private verdictAfterLoad;
133
+ /**
134
+ * The bounded natural wait: poll the live document for challenge markers
135
+ * until they are gone (the user's real browser passed the verification and
136
+ * reloaded into the real page, or swapped it in SPA-style), the per-fetch
137
+ * wait budget runs out, or the fetch's deadline aborts. No clicks, no
138
+ * injected answers — the browser either clears it on its own or it does not.
139
+ *
140
+ * @returns true when the challenge cleared; false when the budget ran out.
141
+ */
142
+ private waitForChallengeClear;
100
143
  }
101
144
  export {};
@@ -12,6 +12,22 @@ export interface PlaywrightResponse {
12
12
  status(): number;
13
13
  headers(): Record<string, string>;
14
14
  text(): Promise<string>;
15
+ /** This response's URL; absent on minimal fakes. */
16
+ url?(): string;
17
+ /**
18
+ * The request this response answers — used to recognize main-frame
19
+ * navigation responses while the challenge wait runs. Absent on fakes.
20
+ */
21
+ request?(): PlaywrightRequest;
22
+ }
23
+ /** The request side of a response, for main-frame filtering. */
24
+ export interface PlaywrightRequest {
25
+ /** True for navigations (document loads and their redirect hops). */
26
+ isNavigationRequest?(): boolean;
27
+ /** `'document'` for frame navigations; absent on minimal fakes. */
28
+ resourceType?(): string;
29
+ /** The frame that issued the request; compare with `page.mainFrame()`. */
30
+ frame?(): unknown;
15
31
  }
16
32
  /** A page inside a context. */
17
33
  export interface PlaywrightPage {
@@ -32,6 +48,19 @@ export interface PlaywrightPage {
32
48
  route(glob: string, handler: (route: PlaywrightRoute) => Promise<void>): Promise<void>;
33
49
  /** Popup notification; the fetch closes whatever its page spawns. */
34
50
  on?(event: 'popup', listener: (page: PlaywrightPage) => void): unknown;
51
+ /**
52
+ * Response notification — the challenge wait uses it to track the LAST
53
+ * main-frame navigation response (challenge pages reload into the real
54
+ * document). Absent on minimal fakes (content polling covers them).
55
+ */
56
+ on?(event: 'response', listener: (response: PlaywrightResponse) => void): unknown;
57
+ /**
58
+ * Evaluate an expression in the page — the challenge probe's live-DOM
59
+ * path. Absent on minimal fakes (content polling covers them).
60
+ */
61
+ evaluate?(script: string, arg?: unknown): Promise<unknown>;
62
+ /** The main frame handle; compare with `request.frame()` for filtering. */
63
+ mainFrame?(): unknown;
35
64
  }
36
65
  /**
37
66
  * A browser context: fetch-owned and isolated (local backend, or CDP
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "dsh-web-fetch-playwright",
3
- "version": "0.2.3",
3
+ "version": "0.2.5",
4
4
  "description": "Playwright/CDP web-fetch provider for DeepSeek Harness: renders pages in a real browser, denoises them (Readability + DOMPurify), and returns markdown.",
5
5
  "type": "module",
6
6
  "license": "MIT",
@@ -31,6 +31,13 @@
31
31
  "README.zh-CN.md",
32
32
  "LICENSE"
33
33
  ],
34
+ "scripts": {
35
+ "build": "rm -rf lib && tsc -p tsconfig.build.json --emitDeclarationOnly && tsdown",
36
+ "prepare": "pnpm run build",
37
+ "prepublishOnly": "pnpm run build",
38
+ "typecheck": "tsc --noEmit",
39
+ "test": "vitest run"
40
+ },
34
41
  "dsh": {
35
42
  "bundle": {
36
43
  "patch": "./cordis.patch.yml"
@@ -120,10 +127,5 @@
120
127
  "tsdown": "^0.22.2",
121
128
  "typescript": "^5.8.0",
122
129
  "vitest": "^4.1.8"
123
- },
124
- "scripts": {
125
- "build": "rm -rf lib && tsc -p tsconfig.build.json --emitDeclarationOnly && tsdown",
126
- "typecheck": "tsc --noEmit",
127
- "test": "vitest run"
128
130
  }
129
- }
131
+ }
@@ -0,0 +1,216 @@
1
+ /**
2
+ * Cloudflare challenge detection: the pure predicates and in-page probe the
3
+ * provider's bounded natural-wait uses. Nothing here talks to a browser,
4
+ * a clock, or the network — every function is a mapping over a response's
5
+ * status/headers or an HTML string, so the whole surface is unit-testable.
6
+ *
7
+ * Detection is layered, strongest signal first:
8
+ *
9
+ * 1. Response-level — Cloudflare documents that EVERY challenge page type
10
+ * (JS challenge, managed challenge, Turnstile interstitial) carries
11
+ * `cf-mitigated: challenge`. That header is the primary verdict; a
12
+ * 403/503 from a `server: cloudflare` HTML response is the fallback for
13
+ * origins that strip it.
14
+ * 2. Content-level — the interstitial's localized `<title>` family
15
+ * ("Just a moment...", "请稍候…", ...) plus structural markers
16
+ * (`/cdn-cgi/challenge-platform/` scripts, `#challenge-*` elements,
17
+ * `cf-chl-widget-` Turnstile frames, `window._cf_chl_opt`). Markers are
18
+ * deliberately structural: a blog post that merely MENTIONS Cloudflare
19
+ * must not classify as a challenge.
20
+ *
21
+ * The security stance (issue #2): this module only DETECTS and classifies.
22
+ * Solving, spoofing, CAPTCHA answering, and cookie lifting live outside the
23
+ * plugin entirely.
24
+ *
25
+ * @module dsh-web-fetch-playwright/challenge
26
+ */
27
+
28
+ /** How often the bounded wait probes the live page for challenge markers. */
29
+ export const CHALLENGE_POLL_INTERVAL_MS = 500
30
+
31
+ /**
32
+ * Tail of the fetch budget held back from the challenge wait so a cleared
33
+ * challenge still has time for its settle (5s networkidle cap) plus the
34
+ * content read and the denoise pipeline.
35
+ */
36
+ export const CHALLENGE_FINISH_RESERVE_MS = 8_000
37
+
38
+ /**
39
+ * The interstitial's localized `<title>` cores, anchored full-match with
40
+ * optional trailing ellipsis/punctuation. Cf. ships dozens of locales; this
41
+ * set covers the major ones — the response header remains the primary
42
+ * signal, so the list only needs to catch header-stripping edges.
43
+ */
44
+ const CHALLENGE_TITLE_CORES = [
45
+ 'just a moment', // en
46
+ 'einen moment', // de
47
+ 'un instant', // fr
48
+ 'un momento', // es, it
49
+ 'um momento', // pt
50
+ 'een ogenblik geduld', // nl
51
+ 'минуточку', 'минутку', 'пожалуйста, подождите', // ru
52
+ 'хвилинку', // uk
53
+ 'chwileczkę', // pl
54
+ 'chvilku strpení', // cs
55
+ 'bir dakika', // tr
56
+ 'o clipă', // ro
57
+ 'μια στιγμή', // el
58
+ '少し時間がかかります', // ja
59
+ '잠시만요', '잠시 기다려주세요', // ko
60
+ '请稍候', '請稍候', // zh-CN, zh-TW
61
+ 'เดี๋ยวก่อน', // th
62
+ 'vui lòng chờ', 'chờ một chút', // vi
63
+ 'tunggu sebentar', // id
64
+ 'لحظة من فضلك', 'الرجاء الانتظار', // ar
65
+ 'רק רגע', // he
66
+ 'ett ögonblick', // sv
67
+ 'et øjeblik', // da
68
+ 'et øyeblikk', // no
69
+ 'hetkinen', 'pikainen hetki', // fi
70
+ 'egy pillanat', // hu
71
+ 'моля, изчакайте', // bg
72
+ ]
73
+
74
+ /** Escape regex metacharacters in a literal title core. */
75
+ function escapeRegExp(text: string): string {
76
+ return text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
77
+ }
78
+
79
+ /**
80
+ * Anchored, case-insensitive match for the localized interstitial title:
81
+ * one of the cores, then only ellipsis/dot/punctuation noise to the end.
82
+ */
83
+ export const CHALLENGE_TITLE_RE = new RegExp(
84
+ `^(?:${CHALLENGE_TITLE_CORES.map(escapeRegExp).join('|')})[\\s.!??…。!]*$`,
85
+ 'i',
86
+ )
87
+
88
+ /**
89
+ * Structural markers a challenge document carries in its HTML. Kept in
90
+ * attribute/assignment form (`id="…"`, `window._cf_chl_opt`) rather than bare
91
+ * substrings so an article that merely QUOTES these strings in prose or code
92
+ * samples does not classify — the marker must look like the real element.
93
+ */
94
+ const CHALLENGE_HTML_MARKERS = [
95
+ '/cdn-cgi/challenge-platform/',
96
+ '/cdn-cgi/scripts/jsd/main.js',
97
+ 'id="cf-chl-widget-',
98
+ 'window._cf_chl_opt',
99
+ 'id="challenge-form"',
100
+ 'id="challenge-running"',
101
+ 'id="challenge-stage"',
102
+ 'id="challenge-error-text"',
103
+ 'name="cf-challenge"',
104
+ 'checking your browser before accessing',
105
+ ]
106
+
107
+ /** Title cores of the hard-block page ("Attention Required! | Cloudflare"). */
108
+ const BLOCKED_TITLE_RE = /^attention required!/i
109
+
110
+ /**
111
+ * The hard-block page's body phrase, gated by the structural markers below:
112
+ * the phrase alone must not fail a normal article that happens to contain
113
+ * it in prose.
114
+ */
115
+ const BLOCKED_BODY_PHRASE = 'you have been blocked'
116
+
117
+ /** Structural markers of the hard-block page's classic layout. */
118
+ const BLOCKED_BODY_MARKERS = ['cf-headline', 'cf-error-details', 'cf-block-details']
119
+
120
+ /** The verdicts detection can return. */
121
+ export type ChallengeVerdict = 'none' | 'challenge' | 'blocked'
122
+
123
+ /** Extract and trim a document's `<title>` text, '' when absent. */
124
+ function titleOf(html: string): string {
125
+ const match = /<title[^>]*>([^<]*)<\/title>/i.exec(html)
126
+ return (match?.[1] ?? '').trim()
127
+ }
128
+
129
+ /** Whether a content-type mime is an HTML document (challenge pages are). */
130
+ function isHtmlMime(contentType: string): boolean {
131
+ const mime = contentType.replace(/;.*$/s, '').trim().toLowerCase()
132
+ return mime === '' || mime === 'text/html' || mime === 'application/xhtml+xml'
133
+ }
134
+
135
+ /**
136
+ * Classify a navigation response from its status and headers.
137
+ *
138
+ * @param status - the response's HTTP status code.
139
+ * @param headers - the response's headers (case as Playwright serves: lowercased).
140
+ * @returns `'challenge'` when the response is a Cloudflare challenge, else `'none'`.
141
+ */
142
+ export function classifyChallengeResponse(status: number, headers: Record<string, string>): 'none' | 'challenge' {
143
+ const header = (name: string): string => (headers[name] ?? '').trim().toLowerCase()
144
+ // The documented signal: every challenge page type sets this header.
145
+ if (header('cf-mitigated') === 'challenge') return 'challenge'
146
+ // Fallback for origins that strip it: Cloudflare's own block statuses
147
+ // served as HTML from a Cloudflare edge.
148
+ if ((status === 403 || status === 503) && header('server').includes('cloudflare') && isHtmlMime(header('content-type'))) {
149
+ return 'challenge'
150
+ }
151
+ return 'none'
152
+ }
153
+
154
+ /**
155
+ * Whether a response is even a candidate for the content-level checks —
156
+ * the suspicion gate. Cloudflare interstitials NEVER ship a plain 200 from
157
+ * a non-Cloudflare origin: they are 403/429/503, or served from a Cloudflare
158
+ * edge (`server: cloudflare`, `cf-ray`). Running the title/marker fallback
159
+ * only on such responses means a normal article — whatever it quotes about
160
+ * Cloudflare, in any language — can never be misread as a challenge. This is
161
+ * the tiering the industry converges on: protocol signals first, content
162
+ * markers only as a gated fallback.
163
+ *
164
+ * @param status - the response's HTTP status code.
165
+ * @param headers - the response's headers (lowercased, as Playwright serves).
166
+ * @returns true when the content-level fallback may run.
167
+ */
168
+ export function isChallengeCompatibleResponse(status: number, headers: Record<string, string>): boolean {
169
+ if (status === 403 || status === 429 || status === 503) return true
170
+ const header = (name: string): string => (headers[name] ?? '').trim().toLowerCase()
171
+ return header('server').includes('cloudflare') || header('cf-ray') !== ''
172
+ }
173
+
174
+ /**
175
+ * Classify a rendered document from its HTML: the localized interstitial
176
+ * title family and the structural markers above say `challenge`; the
177
+ * hard-block page says `blocked` (waiting cannot clear it — the caller
178
+ * should fail immediately instead of burning its budget).
179
+ *
180
+ * @param html - the document HTML as `page.content()` returns it.
181
+ * @returns the verdict for this HTML.
182
+ */
183
+ export function classifyChallengeHtml(html: string): ChallengeVerdict {
184
+ if (html.length === 0) return 'none'
185
+ const title = titleOf(html)
186
+ if (BLOCKED_TITLE_RE.test(title)) return 'blocked'
187
+ const lowered = html.toLowerCase()
188
+ if (lowered.includes(BLOCKED_BODY_PHRASE) && BLOCKED_BODY_MARKERS.some(marker => lowered.includes(marker))) return 'blocked'
189
+ if (CHALLENGE_TITLE_RE.test(title)) return 'challenge'
190
+ if (CHALLENGE_HTML_MARKERS.some(marker => lowered.includes(marker))) return 'challenge'
191
+ // Crawlee's footer marker, as a two-class combo so prose quoting one class
192
+ // alone stays clean.
193
+ if (lowered.includes('class="ray-id"') && lowered.includes('class="footer-inner"')) return 'challenge'
194
+ return 'none'
195
+ }
196
+
197
+ /**
198
+ * The in-page probe the bounded wait runs against the LIVE document (via
199
+ * `page.evaluate`): true while challenge markers are still present. This is
200
+ * what makes SPA-style clears visible — the document swaps its content
201
+ * without any navigation, so only a DOM read can see the change. The title
202
+ * regex is baked in at build time from the same single source the Node-side
203
+ * check uses, so the two can never drift apart.
204
+ */
205
+ export const CHALLENGE_DOM_PROBE = `(() => {
206
+ const re = new RegExp(${JSON.stringify(CHALLENGE_TITLE_RE.source)}, ${JSON.stringify(CHALLENGE_TITLE_RE.flags)})
207
+ if (re.test(document.title.trim())) return true
208
+ if (document.querySelector('#challenge-form, #challenge-running, #challenge-stage, #challenge-error-text')) return true
209
+ if (document.querySelector('[id^="cf-chl-widget-"]')) return true
210
+ for (const script of Array.from(document.scripts)) {
211
+ const src = script.getAttribute('src') || ''
212
+ if (src.includes('/cdn-cgi/challenge-platform/') || src.includes('/cdn-cgi/scripts/jsd/main.js')) return true
213
+ }
214
+ if (document.querySelector('.footer .footer-inner .ray-id')) return true
215
+ return false
216
+ })()`
@@ -141,6 +141,19 @@ export function PlaywrightCard(props: PlaywrightCardProps) {
141
141
  onEdit={(text) => { props.edit('maxConcurrency', text) }}
142
142
  onReset={() => { props.resetField('maxConcurrency') }}
143
143
  />
144
+ <ValueField
145
+ id="plugin-config-playwright-challenge-wait"
146
+ label={t('challengeWaitMs')}
147
+ hint={t('challengeWaitMsHint')}
148
+ placeholder={t('challengeWaitMsPlaceholder')}
149
+ overriddenLabel={t('overridden')}
150
+ resetLabel={t('reset')}
151
+ invalidLabel={t('invalidText')}
152
+ disabled={disabled}
153
+ {...state.challengeWaitMs}
154
+ onEdit={(text) => { props.edit('challengeWaitMs', text) }}
155
+ onReset={() => { props.resetField('challengeWaitMs') }}
156
+ />
144
157
  </PluginCard>
145
158
  )
146
159
  }
@@ -34,6 +34,11 @@ export interface PlaywrightSettings {
34
34
  denoise?: boolean
35
35
  /** How many fetches may render at once (1–200); blank = backend default. */
36
36
  maxConcurrency?: number
37
+ /**
38
+ * Bounded wait (ms) for a Cloudflare challenge to clear naturally in the
39
+ * same tab (0–60000); 0 = off (return the first response as-is).
40
+ */
41
+ challengeWaitMs?: number
37
42
  }
38
43
 
39
44
  /** What the Playwright card renders. */
@@ -50,6 +55,8 @@ export interface PlaywrightCardState extends CardShell {
50
55
  denoise: CardFieldState
51
56
  /** Concurrency input (draft decimal integer). */
52
57
  maxConcurrency: CardFieldState
58
+ /** Challenge wait input (draft decimal integer of milliseconds). */
59
+ challengeWaitMs: CardFieldState
53
60
  }
54
61
 
55
62
  /** The registration-side face the card's slot entry injects. */
@@ -78,6 +85,7 @@ export class PlaywrightCardController {
78
85
  checkboxField('shareBrowserContext'),
79
86
  checkboxField('denoise'),
80
87
  numberField('maxConcurrency', 1, 200),
88
+ numberField('challengeWaitMs', 0, 60_000),
81
89
  ],
82
90
  )
83
91
  this.store = this.form.bind(() => this.projection())
@@ -92,6 +100,7 @@ export class PlaywrightCardController {
92
100
  shareBrowserContext: this.form.field('shareBrowserContext'),
93
101
  denoise: this.form.field('denoise'),
94
102
  maxConcurrency: this.form.field('maxConcurrency'),
103
+ challengeWaitMs: this.form.field('challengeWaitMs'),
95
104
  }
96
105
  }
97
106
 
@@ -14,6 +14,7 @@ export type PlaywrightCardLocaleKey =
14
14
  | 'shareBrowserContext' | 'shareBrowserContextHint'
15
15
  | 'denoise' | 'denoiseHint'
16
16
  | 'maxConcurrency' | 'maxConcurrencyHint' | 'maxConcurrencyPlaceholder'
17
+ | 'challengeWaitMs' | 'challengeWaitMsHint' | 'challengeWaitMsPlaceholder'
17
18
  | 'overridden' | 'reset' | 'readOnly' | 'expand' | 'collapse'
18
19
  | 'save' | 'saving' | 'discard' | 'unsaved' | 'saveFailed' | 'invalidText'
19
20
 
@@ -45,6 +46,9 @@ export const en: Record<PlaywrightCardLocaleKey, string> = {
45
46
  maxConcurrency: 'Max concurrent fetches',
46
47
  maxConcurrencyHint: 'How many pages may render at once (1–200). Blank = auto: 4 local browsers, 50 CDP tabs in the remote browser.',
47
48
  maxConcurrencyPlaceholder: '(auto: local 4 / CDP 50)',
49
+ challengeWaitMs: 'Cloudflare challenge wait (ms)',
50
+ challengeWaitMsHint: 'Bounded wait for a Cloudflare challenge to clear naturally in the same tab (0–60000; 0 = off). 0 disables: the first response is returned as-is.',
51
+ challengeWaitMsPlaceholder: '(default: 15000)',
48
52
  overridden: 'Overridden',
49
53
  reset: 'Reset to default',
50
54
  readOnly: 'This deployment stores settings read-only.',
@@ -79,6 +83,9 @@ export const zh: Record<PlaywrightCardLocaleKey, string> = {
79
83
  maxConcurrency: '最大并发抓取数',
80
84
  maxConcurrencyHint: '同时渲染的页面上限(1–200);留空自动:本地 4 个浏览器,CDP 在远端浏览器里 50 个标签页。',
81
85
  maxConcurrencyPlaceholder: '(自动:本地 4 / CDP 50)',
86
+ challengeWaitMs: 'Cloudflare 挑战等待上限(毫秒)',
87
+ challengeWaitMsHint: '在同一标签页内有界等待 Cloudflare 验证自然通过(0–60000;0 = 关闭)。关闭时直接返回首次响应——旧版行为。',
88
+ challengeWaitMsPlaceholder: '(默认:15000)',
82
89
  overridden: '已覆盖',
83
90
  reset: '恢复默认',
84
91
  readOnly: '本部署的设置为只读。',
package/src/config.ts CHANGED
@@ -30,6 +30,23 @@ export const DEFAULT_MAX_CONCURRENCY_CDP = 50
30
30
  /** Ceiling the schema accepts for `maxConcurrency` (local slots are browsers). */
31
31
  export const MAX_CONCURRENCY_CEILING = 200
32
32
 
33
+ /**
34
+ * Default bounded wait (ms) for a Cloudflare challenge to clear naturally —
35
+ * the user's real browser passes the verification on its own while the fetch
36
+ * holds the same page and context. Sized to fit the 45s per-fetch deadline
37
+ * with the settle/decode tail (and one retry) still inside it.
38
+ */
39
+ export const DEFAULT_CHALLENGE_WAIT_MS = 15_000
40
+
41
+ /** Ceiling the schema accepts for `challengeWaitMs`. */
42
+ export const MAX_CHALLENGE_WAIT_MS = 60_000
43
+
44
+ /** Default same-page re-navigation attempts after a challenge wait runs out. */
45
+ export const DEFAULT_CHALLENGE_RETRIES = 1
46
+
47
+ /** Ceiling the schema accepts for `challengeRetries`. */
48
+ export const MAX_CHALLENGE_RETRIES = 3
49
+
33
50
  /** Which browser backend serves a fetch. */
34
51
  export type PlaywrightBackend = 'local' | 'cdp'
35
52
 
@@ -61,6 +78,19 @@ export interface Config {
61
78
  shareBrowserContext?: boolean
62
79
  /** Whether the Readability + DOMPurify denoise pipeline runs before markdown. */
63
80
  denoise?: boolean
81
+ /**
82
+ * Bounded wait (ms) for a Cloudflare challenge to clear naturally inside
83
+ * the same page/context. `0` (or any config leaving this at 0) restores the
84
+ * legacy behavior: the first response is final, no waiting — the A/B
85
+ * baseline. Schema default: {@link DEFAULT_CHALLENGE_WAIT_MS}.
86
+ */
87
+ challengeWaitMs?: number
88
+ /**
89
+ * Same-page re-navigation attempts after a challenge wait window runs out
90
+ * (any partial clearance cookies stay in the context for the retry).
91
+ * Schema default: {@link DEFAULT_CHALLENGE_RETRIES}.
92
+ */
93
+ challengeRetries?: number
64
94
  /**
65
95
  * How many fetches may render at once. Blank = backend default (4 for
66
96
  * local — each slot launches a browser; 50 for CDP — each slot is a tab in
@@ -81,6 +111,9 @@ export const Config: z<Config> = z.object({
81
111
  // Optional on purpose: the effective default depends on `backend`, which a
82
112
  // static schema default cannot express.
83
113
  maxConcurrency: z.number().step(1).min(1).max(MAX_CONCURRENCY_CEILING),
114
+ // The bounded natural-wait knobs; 0 disables the whole challenge path.
115
+ challengeWaitMs: z.number().step(100).min(0).max(MAX_CHALLENGE_WAIT_MS).default(DEFAULT_CHALLENGE_WAIT_MS),
116
+ challengeRetries: z.number().step(1).min(0).max(MAX_CHALLENGE_RETRIES).default(DEFAULT_CHALLENGE_RETRIES),
84
117
  })
85
118
 
86
119
  /**
@@ -103,6 +136,30 @@ export function effectiveMaxConcurrency(config: Pick<Config, 'backend' | 'maxCon
103
136
  return config.backend === 'cdp' ? DEFAULT_MAX_CONCURRENCY_CDP : DEFAULT_MAX_CONCURRENCY_LOCAL
104
137
  }
105
138
 
139
+ /**
140
+ * The challenge wait budget a fetch actually runs with: an explicit setting
141
+ * wins, else the schema default. `0` means the challenge path is off.
142
+ *
143
+ * @param config - the resolved settings section (or any partial of it).
144
+ * @returns the bounded wait in milliseconds.
145
+ */
146
+ export function effectiveChallengeWaitMs(config: Pick<Config, 'challengeWaitMs'>): number {
147
+ if (typeof config.challengeWaitMs === 'number') return config.challengeWaitMs
148
+ return DEFAULT_CHALLENGE_WAIT_MS
149
+ }
150
+
151
+ /**
152
+ * The same-page retry count a challenge fetch actually runs with: an explicit
153
+ * setting wins, else the schema default.
154
+ *
155
+ * @param config - the resolved settings section (or any partial of it).
156
+ * @returns retries after a wait window runs out.
157
+ */
158
+ export function effectiveChallengeRetries(config: Pick<Config, 'challengeRetries'>): number {
159
+ if (typeof config.challengeRetries === 'number') return config.challengeRetries
160
+ return DEFAULT_CHALLENGE_RETRIES
161
+ }
162
+
106
163
  /**
107
164
  * The context mode a fetch actually runs with: profile (a tab in the remote
108
165
  * browser's real profile) requires the CDP backend — a local launch has no
package/src/index.ts CHANGED
@@ -17,11 +17,13 @@ import { Config } from './config.ts'
17
17
  import type { ResolvedConfig } from './config.ts'
18
18
  import { PlaywrightFetchProvider } from './provider.ts'
19
19
 
20
- export { Config, DEFAULT_CDP_ENDPOINT, effectiveContextMode, normalizeCdpEndpoint } from './config.ts'
20
+ export { Config, DEFAULT_CDP_ENDPOINT, DEFAULT_CHALLENGE_RETRIES, DEFAULT_CHALLENGE_WAIT_MS, effectiveChallengeRetries, effectiveChallengeWaitMs, effectiveContextMode, normalizeCdpEndpoint } from './config.ts'
21
21
  export type { Config as PlaywrightFetchConfig, CdpContextMode, PlaywrightBackend, ResolvedConfig } from './config.ts'
22
22
  export { CdpConnectionPool } from './cdp-pool.ts'
23
23
  export type { CdpAcquireMode, CdpConnect, CdpLease } from './cdp-pool.ts'
24
- export { PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider } from './provider.ts'
24
+ export { CHALLENGE_DOM_PROBE, CHALLENGE_FINISH_RESERVE_MS, CHALLENGE_POLL_INTERVAL_MS, CHALLENGE_TITLE_RE, classifyChallengeHtml, classifyChallengeResponse, isChallengeCompatibleResponse } from './challenge.ts'
25
+ export type { ChallengeVerdict } from './challenge.ts'
26
+ export { PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider, WEB_FETCH_CHALLENGE_CODE } from './provider.ts'
25
27
  export { htmlToMarkdown } from './markdown.ts'
26
28
  export type { DenoiseMode, DenoiseResult } from './markdown.ts'
27
29