dsh-web-fetch-playwright 0.2.3 → 0.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -2
- package/README.zh-CN.md +18 -2
- package/dsh.plugin.json +1 -1
- package/lib/client.js +52 -27
- package/lib/index.js +451 -15
- package/lib/types/challenge.d.ts +83 -0
- package/lib/types/client/controller.d.ts +7 -0
- package/lib/types/client/locales.d.ts +1 -1
- package/lib/types/config.d.ts +42 -0
- package/lib/types/index.d.ts +4 -2
- package/lib/types/markdown.d.ts +2 -0
- package/lib/types/provider.d.ts +44 -1
- package/lib/types/types.d.ts +29 -0
- package/package.json +9 -7
- package/src/challenge.ts +216 -0
- package/src/client/card.tsx +13 -0
- package/src/client/controller.ts +9 -0
- package/src/client/locales.ts +7 -0
- package/src/config.ts +57 -0
- package/src/index.ts +4 -2
- package/src/markdown.ts +55 -0
- package/src/provider.ts +223 -10
- package/src/types.ts +30 -0
package/lib/types/config.d.ts
CHANGED
|
@@ -24,6 +24,19 @@ export declare const DEFAULT_MAX_CONCURRENCY_LOCAL = 4;
|
|
|
24
24
|
export declare const DEFAULT_MAX_CONCURRENCY_CDP = 50;
|
|
25
25
|
/** Ceiling the schema accepts for `maxConcurrency` (local slots are browsers). */
|
|
26
26
|
export declare const MAX_CONCURRENCY_CEILING = 200;
|
|
27
|
+
/**
|
|
28
|
+
* Default bounded wait (ms) for a Cloudflare challenge to clear naturally —
|
|
29
|
+
* the user's real browser passes the verification on its own while the fetch
|
|
30
|
+
* holds the same page and context. Sized to fit the 45s per-fetch deadline
|
|
31
|
+
* with the settle/decode tail (and one retry) still inside it.
|
|
32
|
+
*/
|
|
33
|
+
export declare const DEFAULT_CHALLENGE_WAIT_MS = 15000;
|
|
34
|
+
/** Ceiling the schema accepts for `challengeWaitMs`. */
|
|
35
|
+
export declare const MAX_CHALLENGE_WAIT_MS = 60000;
|
|
36
|
+
/** Default same-page re-navigation attempts after a challenge wait runs out. */
|
|
37
|
+
export declare const DEFAULT_CHALLENGE_RETRIES = 1;
|
|
38
|
+
/** Ceiling the schema accepts for `challengeRetries`. */
|
|
39
|
+
export declare const MAX_CHALLENGE_RETRIES = 3;
|
|
27
40
|
/** Which browser backend serves a fetch. */
|
|
28
41
|
export type PlaywrightBackend = 'local' | 'cdp';
|
|
29
42
|
/**
|
|
@@ -53,6 +66,19 @@ export interface Config {
|
|
|
53
66
|
shareBrowserContext?: boolean;
|
|
54
67
|
/** Whether the Readability + DOMPurify denoise pipeline runs before markdown. */
|
|
55
68
|
denoise?: boolean;
|
|
69
|
+
/**
|
|
70
|
+
* Bounded wait (ms) for a Cloudflare challenge to clear naturally inside
|
|
71
|
+
* the same page/context. `0` (or any config leaving this at 0) restores the
|
|
72
|
+
* legacy behavior: the first response is final, no waiting — the A/B
|
|
73
|
+
* baseline. Schema default: {@link DEFAULT_CHALLENGE_WAIT_MS}.
|
|
74
|
+
*/
|
|
75
|
+
challengeWaitMs?: number;
|
|
76
|
+
/**
|
|
77
|
+
* Same-page re-navigation attempts after a challenge wait window runs out
|
|
78
|
+
* (any partial clearance cookies stay in the context for the retry).
|
|
79
|
+
* Schema default: {@link DEFAULT_CHALLENGE_RETRIES}.
|
|
80
|
+
*/
|
|
81
|
+
challengeRetries?: number;
|
|
56
82
|
/**
|
|
57
83
|
* How many fetches may render at once. Blank = backend default (4 for
|
|
58
84
|
* local — each slot launches a browser; 50 for CDP — each slot is a tab in
|
|
@@ -78,6 +104,22 @@ export type ResolvedConfig = Omit<Required<Config>, 'maxConcurrency'> & {
|
|
|
78
104
|
* @returns the effective limit for the semaphore.
|
|
79
105
|
*/
|
|
80
106
|
export declare function effectiveMaxConcurrency(config: Pick<Config, 'backend' | 'maxConcurrency'>): number;
|
|
107
|
+
/**
|
|
108
|
+
* The challenge wait budget a fetch actually runs with: an explicit setting
|
|
109
|
+
* wins, else the schema default. `0` means the challenge path is off.
|
|
110
|
+
*
|
|
111
|
+
* @param config - the resolved settings section (or any partial of it).
|
|
112
|
+
* @returns the bounded wait in milliseconds.
|
|
113
|
+
*/
|
|
114
|
+
export declare function effectiveChallengeWaitMs(config: Pick<Config, 'challengeWaitMs'>): number;
|
|
115
|
+
/**
|
|
116
|
+
* The same-page retry count a challenge fetch actually runs with: an explicit
|
|
117
|
+
* setting wins, else the schema default.
|
|
118
|
+
*
|
|
119
|
+
* @param config - the resolved settings section (or any partial of it).
|
|
120
|
+
* @returns retries after a wait window runs out.
|
|
121
|
+
*/
|
|
122
|
+
export declare function effectiveChallengeRetries(config: Pick<Config, 'challengeRetries'>): number;
|
|
81
123
|
/**
|
|
82
124
|
* The context mode a fetch actually runs with: profile (a tab in the remote
|
|
83
125
|
* browser's real profile) requires the CDP backend — a local launch has no
|
package/lib/types/index.d.ts
CHANGED
|
@@ -11,11 +11,13 @@
|
|
|
11
11
|
*/
|
|
12
12
|
import type { Context } from '@deepseek-ai/cordis';
|
|
13
13
|
import { Config } from './config.ts';
|
|
14
|
-
export { Config, DEFAULT_CDP_ENDPOINT, effectiveContextMode, normalizeCdpEndpoint } from './config.ts';
|
|
14
|
+
export { Config, DEFAULT_CDP_ENDPOINT, DEFAULT_CHALLENGE_RETRIES, DEFAULT_CHALLENGE_WAIT_MS, effectiveChallengeRetries, effectiveChallengeWaitMs, effectiveContextMode, normalizeCdpEndpoint } from './config.ts';
|
|
15
15
|
export type { Config as PlaywrightFetchConfig, CdpContextMode, PlaywrightBackend, ResolvedConfig } from './config.ts';
|
|
16
16
|
export { CdpConnectionPool } from './cdp-pool.ts';
|
|
17
17
|
export type { CdpAcquireMode, CdpConnect, CdpLease } from './cdp-pool.ts';
|
|
18
|
-
export {
|
|
18
|
+
export { CHALLENGE_DOM_PROBE, CHALLENGE_FINISH_RESERVE_MS, CHALLENGE_POLL_INTERVAL_MS, CHALLENGE_TITLE_RE, classifyChallengeHtml, classifyChallengeResponse, isChallengeCompatibleResponse } from './challenge.ts';
|
|
19
|
+
export type { ChallengeVerdict } from './challenge.ts';
|
|
20
|
+
export { PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider, WEB_FETCH_CHALLENGE_CODE } from './provider.ts';
|
|
19
21
|
export { htmlToMarkdown } from './markdown.ts';
|
|
20
22
|
export type { DenoiseMode, DenoiseResult } from './markdown.ts';
|
|
21
23
|
/** Cordis plugin name used by loader diagnostics. */
|
package/lib/types/markdown.d.ts
CHANGED
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
* The denoise pipeline: rendered HTML → sanitized article HTML → markdown.
|
|
3
3
|
*
|
|
4
4
|
* The classic stack, in order: jsdom parses the page Playwright rendered;
|
|
5
|
+
* inline `data:` image payloads are elided to size placeholders (build
|
|
6
|
+
* tools inline images as base64, which would otherwise dominate the body);
|
|
5
7
|
* Mozilla Readability extracts the article (dropping nav bars, sidebars,
|
|
6
8
|
* footers, and ad chrome by scoring link density and text mass); DOMPurify
|
|
7
9
|
* sanitizes whatever HTML remains and forbids the layout tags noise lives in;
|
package/lib/types/provider.d.ts
CHANGED
|
@@ -22,6 +22,18 @@
|
|
|
22
22
|
* browser can reach. Profile mode additionally acts WITH the remote
|
|
23
23
|
* browser's logged-in sessions (see the README's risk notes).
|
|
24
24
|
*
|
|
25
|
+
* Cloudflare challenges (issue #2): when a navigation lands on a challenge
|
|
26
|
+
* interstitial, the fetch waits — on the SAME page and in the SAME browser
|
|
27
|
+
* context, so the browser's natural verification and any clearance cookies
|
|
28
|
+
* it earns apply — for a bounded, configurable window
|
|
29
|
+
* (`challengeWaitMs`, default 15s; 0 restores the legacy first-response
|
|
30
|
+
* behavior). The wait tracks the LAST main-frame navigation response (the
|
|
31
|
+
* real page reloads in after the challenge clears) and watches the live DOM
|
|
32
|
+
* so SPA-style clears are caught too. It never clicks, never injects
|
|
33
|
+
* answers, never touches cookies itself; when the budget runs out the fetch
|
|
34
|
+
* fails with {@link WEB_FETCH_CHALLENGE_CODE} instead of returning the
|
|
35
|
+
* interstitial as content.
|
|
36
|
+
*
|
|
25
37
|
* @module dsh-web-fetch-playwright/provider
|
|
26
38
|
*/
|
|
27
39
|
import type { WebFetchProvider, WebFetchRequest, WebFetchResult } from '@deepseek-ai/dsh-web';
|
|
@@ -30,6 +42,14 @@ import { CdpConnectionPool } from './cdp-pool.ts';
|
|
|
30
42
|
import type { PlaywrightBrowser, PlaywrightContext, PlaywrightPage } from './types.ts';
|
|
31
43
|
/** Stable id this provider registers under (the bundle patch pins it). */
|
|
32
44
|
export declare const PLAYWRIGHT_FETCH_PROVIDER_ID = "playwright";
|
|
45
|
+
/**
|
|
46
|
+
* Error code for a Cloudflare challenge the bounded natural wait could not
|
|
47
|
+
* clear. Provider-specific by design — the web seam's `code` is an open
|
|
48
|
+
* string and consumers must tolerate provider-specific codes — so callers
|
|
49
|
+
* can tell "the site challenged us and the browser did not pass" apart from
|
|
50
|
+
* a transport timeout or a provider bug.
|
|
51
|
+
*/
|
|
52
|
+
export declare const WEB_FETCH_CHALLENGE_CODE = "WEB_FETCH_CHALLENGE";
|
|
33
53
|
/**
|
|
34
54
|
* One render session: a browser, the context this fetch works in, and the
|
|
35
55
|
* tab it owns. Local backends own the browser too; CDP sessions ride the
|
|
@@ -95,7 +115,30 @@ export declare class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
95
115
|
* @returns the browser session the fetch will use.
|
|
96
116
|
*/
|
|
97
117
|
protected openSession(config: ResolvedConfig, deadline: Deadline): Promise<BrowserSession>;
|
|
98
|
-
/**
|
|
118
|
+
/**
|
|
119
|
+
* Navigate, settle, and decode one URL inside an open session's tab —
|
|
120
|
+
* waiting out any Cloudflare interstitial within the bounded challenge
|
|
121
|
+
* budget before the final document is read (see the class docs).
|
|
122
|
+
*/
|
|
99
123
|
private retrieve;
|
|
124
|
+
/**
|
|
125
|
+
* Whether the freshly loaded document is a Cloudflare interstitial: the
|
|
126
|
+
* response-level signals first (the documented `cf-mitigated` header, then
|
|
127
|
+
* the 403/503 + cloudflare fallback), then — gated behind
|
|
128
|
+
* {@link isChallengeCompatibleResponse}, because interstitials never ship
|
|
129
|
+
* a plain 200 — the localized title family and structural markers, which
|
|
130
|
+
* also recognize the hard-block page waiting can never clear.
|
|
131
|
+
*/
|
|
132
|
+
private verdictAfterLoad;
|
|
133
|
+
/**
|
|
134
|
+
* The bounded natural wait: poll the live document for challenge markers
|
|
135
|
+
* until they are gone (the user's real browser passed the verification and
|
|
136
|
+
* reloaded into the real page, or swapped it in SPA-style), the per-fetch
|
|
137
|
+
* wait budget runs out, or the fetch's deadline aborts. No clicks, no
|
|
138
|
+
* injected answers — the browser either clears it on its own or it does not.
|
|
139
|
+
*
|
|
140
|
+
* @returns true when the challenge cleared; false when the budget ran out.
|
|
141
|
+
*/
|
|
142
|
+
private waitForChallengeClear;
|
|
100
143
|
}
|
|
101
144
|
export {};
|
package/lib/types/types.d.ts
CHANGED
|
@@ -12,6 +12,22 @@ export interface PlaywrightResponse {
|
|
|
12
12
|
status(): number;
|
|
13
13
|
headers(): Record<string, string>;
|
|
14
14
|
text(): Promise<string>;
|
|
15
|
+
/** This response's URL; absent on minimal fakes. */
|
|
16
|
+
url?(): string;
|
|
17
|
+
/**
|
|
18
|
+
* The request this response answers — used to recognize main-frame
|
|
19
|
+
* navigation responses while the challenge wait runs. Absent on fakes.
|
|
20
|
+
*/
|
|
21
|
+
request?(): PlaywrightRequest;
|
|
22
|
+
}
|
|
23
|
+
/** The request side of a response, for main-frame filtering. */
|
|
24
|
+
export interface PlaywrightRequest {
|
|
25
|
+
/** True for navigations (document loads and their redirect hops). */
|
|
26
|
+
isNavigationRequest?(): boolean;
|
|
27
|
+
/** `'document'` for frame navigations; absent on minimal fakes. */
|
|
28
|
+
resourceType?(): string;
|
|
29
|
+
/** The frame that issued the request; compare with `page.mainFrame()`. */
|
|
30
|
+
frame?(): unknown;
|
|
15
31
|
}
|
|
16
32
|
/** A page inside a context. */
|
|
17
33
|
export interface PlaywrightPage {
|
|
@@ -32,6 +48,19 @@ export interface PlaywrightPage {
|
|
|
32
48
|
route(glob: string, handler: (route: PlaywrightRoute) => Promise<void>): Promise<void>;
|
|
33
49
|
/** Popup notification; the fetch closes whatever its page spawns. */
|
|
34
50
|
on?(event: 'popup', listener: (page: PlaywrightPage) => void): unknown;
|
|
51
|
+
/**
|
|
52
|
+
* Response notification — the challenge wait uses it to track the LAST
|
|
53
|
+
* main-frame navigation response (challenge pages reload into the real
|
|
54
|
+
* document). Absent on minimal fakes (content polling covers them).
|
|
55
|
+
*/
|
|
56
|
+
on?(event: 'response', listener: (response: PlaywrightResponse) => void): unknown;
|
|
57
|
+
/**
|
|
58
|
+
* Evaluate an expression in the page — the challenge probe's live-DOM
|
|
59
|
+
* path. Absent on minimal fakes (content polling covers them).
|
|
60
|
+
*/
|
|
61
|
+
evaluate?(script: string, arg?: unknown): Promise<unknown>;
|
|
62
|
+
/** The main frame handle; compare with `request.frame()` for filtering. */
|
|
63
|
+
mainFrame?(): unknown;
|
|
35
64
|
}
|
|
36
65
|
/**
|
|
37
66
|
* A browser context: fetch-owned and isolated (local backend, or CDP
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dsh-web-fetch-playwright",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.5",
|
|
4
4
|
"description": "Playwright/CDP web-fetch provider for DeepSeek Harness: renders pages in a real browser, denoises them (Readability + DOMPurify), and returns markdown.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -31,6 +31,13 @@
|
|
|
31
31
|
"README.zh-CN.md",
|
|
32
32
|
"LICENSE"
|
|
33
33
|
],
|
|
34
|
+
"scripts": {
|
|
35
|
+
"build": "rm -rf lib && tsc -p tsconfig.build.json --emitDeclarationOnly && tsdown",
|
|
36
|
+
"prepare": "pnpm run build",
|
|
37
|
+
"prepublishOnly": "pnpm run build",
|
|
38
|
+
"typecheck": "tsc --noEmit",
|
|
39
|
+
"test": "vitest run"
|
|
40
|
+
},
|
|
34
41
|
"dsh": {
|
|
35
42
|
"bundle": {
|
|
36
43
|
"patch": "./cordis.patch.yml"
|
|
@@ -120,10 +127,5 @@
|
|
|
120
127
|
"tsdown": "^0.22.2",
|
|
121
128
|
"typescript": "^5.8.0",
|
|
122
129
|
"vitest": "^4.1.8"
|
|
123
|
-
},
|
|
124
|
-
"scripts": {
|
|
125
|
-
"build": "rm -rf lib && tsc -p tsconfig.build.json --emitDeclarationOnly && tsdown",
|
|
126
|
-
"typecheck": "tsc --noEmit",
|
|
127
|
-
"test": "vitest run"
|
|
128
130
|
}
|
|
129
|
-
}
|
|
131
|
+
}
|
package/src/challenge.ts
ADDED
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cloudflare challenge detection: the pure predicates and in-page probe the
|
|
3
|
+
* provider's bounded natural-wait uses. Nothing here talks to a browser,
|
|
4
|
+
* a clock, or the network — every function is a mapping over a response's
|
|
5
|
+
* status/headers or an HTML string, so the whole surface is unit-testable.
|
|
6
|
+
*
|
|
7
|
+
* Detection is layered, strongest signal first:
|
|
8
|
+
*
|
|
9
|
+
* 1. Response-level — Cloudflare documents that EVERY challenge page type
|
|
10
|
+
* (JS challenge, managed challenge, Turnstile interstitial) carries
|
|
11
|
+
* `cf-mitigated: challenge`. That header is the primary verdict; a
|
|
12
|
+
* 403/503 from a `server: cloudflare` HTML response is the fallback for
|
|
13
|
+
* origins that strip it.
|
|
14
|
+
* 2. Content-level — the interstitial's localized `<title>` family
|
|
15
|
+
* ("Just a moment...", "请稍候…", ...) plus structural markers
|
|
16
|
+
* (`/cdn-cgi/challenge-platform/` scripts, `#challenge-*` elements,
|
|
17
|
+
* `cf-chl-widget-` Turnstile frames, `window._cf_chl_opt`). Markers are
|
|
18
|
+
* deliberately structural: a blog post that merely MENTIONS Cloudflare
|
|
19
|
+
* must not classify as a challenge.
|
|
20
|
+
*
|
|
21
|
+
* The security stance (issue #2): this module only DETECTS and classifies.
|
|
22
|
+
* Solving, spoofing, CAPTCHA answering, and cookie lifting live outside the
|
|
23
|
+
* plugin entirely.
|
|
24
|
+
*
|
|
25
|
+
* @module dsh-web-fetch-playwright/challenge
|
|
26
|
+
*/
|
|
27
|
+
|
|
28
|
+
/** How often the bounded wait probes the live page for challenge markers. */
|
|
29
|
+
export const CHALLENGE_POLL_INTERVAL_MS = 500
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Tail of the fetch budget held back from the challenge wait so a cleared
|
|
33
|
+
* challenge still has time for its settle (5s networkidle cap) plus the
|
|
34
|
+
* content read and the denoise pipeline.
|
|
35
|
+
*/
|
|
36
|
+
export const CHALLENGE_FINISH_RESERVE_MS = 8_000
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* The interstitial's localized `<title>` cores, anchored full-match with
|
|
40
|
+
* optional trailing ellipsis/punctuation. Cf. ships dozens of locales; this
|
|
41
|
+
* set covers the major ones — the response header remains the primary
|
|
42
|
+
* signal, so the list only needs to catch header-stripping edges.
|
|
43
|
+
*/
|
|
44
|
+
const CHALLENGE_TITLE_CORES = [
|
|
45
|
+
'just a moment', // en
|
|
46
|
+
'einen moment', // de
|
|
47
|
+
'un instant', // fr
|
|
48
|
+
'un momento', // es, it
|
|
49
|
+
'um momento', // pt
|
|
50
|
+
'een ogenblik geduld', // nl
|
|
51
|
+
'минуточку', 'минутку', 'пожалуйста, подождите', // ru
|
|
52
|
+
'хвилинку', // uk
|
|
53
|
+
'chwileczkę', // pl
|
|
54
|
+
'chvilku strpení', // cs
|
|
55
|
+
'bir dakika', // tr
|
|
56
|
+
'o clipă', // ro
|
|
57
|
+
'μια στιγμή', // el
|
|
58
|
+
'少し時間がかかります', // ja
|
|
59
|
+
'잠시만요', '잠시 기다려주세요', // ko
|
|
60
|
+
'请稍候', '請稍候', // zh-CN, zh-TW
|
|
61
|
+
'เดี๋ยวก่อน', // th
|
|
62
|
+
'vui lòng chờ', 'chờ một chút', // vi
|
|
63
|
+
'tunggu sebentar', // id
|
|
64
|
+
'لحظة من فضلك', 'الرجاء الانتظار', // ar
|
|
65
|
+
'רק רגע', // he
|
|
66
|
+
'ett ögonblick', // sv
|
|
67
|
+
'et øjeblik', // da
|
|
68
|
+
'et øyeblikk', // no
|
|
69
|
+
'hetkinen', 'pikainen hetki', // fi
|
|
70
|
+
'egy pillanat', // hu
|
|
71
|
+
'моля, изчакайте', // bg
|
|
72
|
+
]
|
|
73
|
+
|
|
74
|
+
/** Escape regex metacharacters in a literal title core. */
|
|
75
|
+
function escapeRegExp(text: string): string {
|
|
76
|
+
return text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Anchored, case-insensitive match for the localized interstitial title:
|
|
81
|
+
* one of the cores, then only ellipsis/dot/punctuation noise to the end.
|
|
82
|
+
*/
|
|
83
|
+
export const CHALLENGE_TITLE_RE = new RegExp(
|
|
84
|
+
`^(?:${CHALLENGE_TITLE_CORES.map(escapeRegExp).join('|')})[\\s.!??…。!]*$`,
|
|
85
|
+
'i',
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Structural markers a challenge document carries in its HTML. Kept in
|
|
90
|
+
* attribute/assignment form (`id="…"`, `window._cf_chl_opt`) rather than bare
|
|
91
|
+
* substrings so an article that merely QUOTES these strings in prose or code
|
|
92
|
+
* samples does not classify — the marker must look like the real element.
|
|
93
|
+
*/
|
|
94
|
+
const CHALLENGE_HTML_MARKERS = [
|
|
95
|
+
'/cdn-cgi/challenge-platform/',
|
|
96
|
+
'/cdn-cgi/scripts/jsd/main.js',
|
|
97
|
+
'id="cf-chl-widget-',
|
|
98
|
+
'window._cf_chl_opt',
|
|
99
|
+
'id="challenge-form"',
|
|
100
|
+
'id="challenge-running"',
|
|
101
|
+
'id="challenge-stage"',
|
|
102
|
+
'id="challenge-error-text"',
|
|
103
|
+
'name="cf-challenge"',
|
|
104
|
+
'checking your browser before accessing',
|
|
105
|
+
]
|
|
106
|
+
|
|
107
|
+
/** Title cores of the hard-block page ("Attention Required! | Cloudflare"). */
|
|
108
|
+
const BLOCKED_TITLE_RE = /^attention required!/i
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* The hard-block page's body phrase, gated by the structural markers below:
|
|
112
|
+
* the phrase alone must not fail a normal article that happens to contain
|
|
113
|
+
* it in prose.
|
|
114
|
+
*/
|
|
115
|
+
const BLOCKED_BODY_PHRASE = 'you have been blocked'
|
|
116
|
+
|
|
117
|
+
/** Structural markers of the hard-block page's classic layout. */
|
|
118
|
+
const BLOCKED_BODY_MARKERS = ['cf-headline', 'cf-error-details', 'cf-block-details']
|
|
119
|
+
|
|
120
|
+
/** The verdicts detection can return. */
|
|
121
|
+
export type ChallengeVerdict = 'none' | 'challenge' | 'blocked'
|
|
122
|
+
|
|
123
|
+
/** Extract and trim a document's `<title>` text, '' when absent. */
|
|
124
|
+
function titleOf(html: string): string {
|
|
125
|
+
const match = /<title[^>]*>([^<]*)<\/title>/i.exec(html)
|
|
126
|
+
return (match?.[1] ?? '').trim()
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/** Whether a content-type mime is an HTML document (challenge pages are). */
|
|
130
|
+
function isHtmlMime(contentType: string): boolean {
|
|
131
|
+
const mime = contentType.replace(/;.*$/s, '').trim().toLowerCase()
|
|
132
|
+
return mime === '' || mime === 'text/html' || mime === 'application/xhtml+xml'
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* Classify a navigation response from its status and headers.
|
|
137
|
+
*
|
|
138
|
+
* @param status - the response's HTTP status code.
|
|
139
|
+
* @param headers - the response's headers (case as Playwright serves: lowercased).
|
|
140
|
+
* @returns `'challenge'` when the response is a Cloudflare challenge, else `'none'`.
|
|
141
|
+
*/
|
|
142
|
+
export function classifyChallengeResponse(status: number, headers: Record<string, string>): 'none' | 'challenge' {
|
|
143
|
+
const header = (name: string): string => (headers[name] ?? '').trim().toLowerCase()
|
|
144
|
+
// The documented signal: every challenge page type sets this header.
|
|
145
|
+
if (header('cf-mitigated') === 'challenge') return 'challenge'
|
|
146
|
+
// Fallback for origins that strip it: Cloudflare's own block statuses
|
|
147
|
+
// served as HTML from a Cloudflare edge.
|
|
148
|
+
if ((status === 403 || status === 503) && header('server').includes('cloudflare') && isHtmlMime(header('content-type'))) {
|
|
149
|
+
return 'challenge'
|
|
150
|
+
}
|
|
151
|
+
return 'none'
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* Whether a response is even a candidate for the content-level checks —
|
|
156
|
+
* the suspicion gate. Cloudflare interstitials NEVER ship a plain 200 from
|
|
157
|
+
* a non-Cloudflare origin: they are 403/429/503, or served from a Cloudflare
|
|
158
|
+
* edge (`server: cloudflare`, `cf-ray`). Running the title/marker fallback
|
|
159
|
+
* only on such responses means a normal article — whatever it quotes about
|
|
160
|
+
* Cloudflare, in any language — can never be misread as a challenge. This is
|
|
161
|
+
* the tiering the industry converges on: protocol signals first, content
|
|
162
|
+
* markers only as a gated fallback.
|
|
163
|
+
*
|
|
164
|
+
* @param status - the response's HTTP status code.
|
|
165
|
+
* @param headers - the response's headers (lowercased, as Playwright serves).
|
|
166
|
+
* @returns true when the content-level fallback may run.
|
|
167
|
+
*/
|
|
168
|
+
export function isChallengeCompatibleResponse(status: number, headers: Record<string, string>): boolean {
|
|
169
|
+
if (status === 403 || status === 429 || status === 503) return true
|
|
170
|
+
const header = (name: string): string => (headers[name] ?? '').trim().toLowerCase()
|
|
171
|
+
return header('server').includes('cloudflare') || header('cf-ray') !== ''
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/**
|
|
175
|
+
* Classify a rendered document from its HTML: the localized interstitial
|
|
176
|
+
* title family and the structural markers above say `challenge`; the
|
|
177
|
+
* hard-block page says `blocked` (waiting cannot clear it — the caller
|
|
178
|
+
* should fail immediately instead of burning its budget).
|
|
179
|
+
*
|
|
180
|
+
* @param html - the document HTML as `page.content()` returns it.
|
|
181
|
+
* @returns the verdict for this HTML.
|
|
182
|
+
*/
|
|
183
|
+
export function classifyChallengeHtml(html: string): ChallengeVerdict {
|
|
184
|
+
if (html.length === 0) return 'none'
|
|
185
|
+
const title = titleOf(html)
|
|
186
|
+
if (BLOCKED_TITLE_RE.test(title)) return 'blocked'
|
|
187
|
+
const lowered = html.toLowerCase()
|
|
188
|
+
if (lowered.includes(BLOCKED_BODY_PHRASE) && BLOCKED_BODY_MARKERS.some(marker => lowered.includes(marker))) return 'blocked'
|
|
189
|
+
if (CHALLENGE_TITLE_RE.test(title)) return 'challenge'
|
|
190
|
+
if (CHALLENGE_HTML_MARKERS.some(marker => lowered.includes(marker))) return 'challenge'
|
|
191
|
+
// Crawlee's footer marker, as a two-class combo so prose quoting one class
|
|
192
|
+
// alone stays clean.
|
|
193
|
+
if (lowered.includes('class="ray-id"') && lowered.includes('class="footer-inner"')) return 'challenge'
|
|
194
|
+
return 'none'
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/**
|
|
198
|
+
* The in-page probe the bounded wait runs against the LIVE document (via
|
|
199
|
+
* `page.evaluate`): true while challenge markers are still present. This is
|
|
200
|
+
* what makes SPA-style clears visible — the document swaps its content
|
|
201
|
+
* without any navigation, so only a DOM read can see the change. The title
|
|
202
|
+
* regex is baked in at build time from the same single source the Node-side
|
|
203
|
+
* check uses, so the two can never drift apart.
|
|
204
|
+
*/
|
|
205
|
+
export const CHALLENGE_DOM_PROBE = `(() => {
|
|
206
|
+
const re = new RegExp(${JSON.stringify(CHALLENGE_TITLE_RE.source)}, ${JSON.stringify(CHALLENGE_TITLE_RE.flags)})
|
|
207
|
+
if (re.test(document.title.trim())) return true
|
|
208
|
+
if (document.querySelector('#challenge-form, #challenge-running, #challenge-stage, #challenge-error-text')) return true
|
|
209
|
+
if (document.querySelector('[id^="cf-chl-widget-"]')) return true
|
|
210
|
+
for (const script of Array.from(document.scripts)) {
|
|
211
|
+
const src = script.getAttribute('src') || ''
|
|
212
|
+
if (src.includes('/cdn-cgi/challenge-platform/') || src.includes('/cdn-cgi/scripts/jsd/main.js')) return true
|
|
213
|
+
}
|
|
214
|
+
if (document.querySelector('.footer .footer-inner .ray-id')) return true
|
|
215
|
+
return false
|
|
216
|
+
})()`
|
package/src/client/card.tsx
CHANGED
|
@@ -141,6 +141,19 @@ export function PlaywrightCard(props: PlaywrightCardProps) {
|
|
|
141
141
|
onEdit={(text) => { props.edit('maxConcurrency', text) }}
|
|
142
142
|
onReset={() => { props.resetField('maxConcurrency') }}
|
|
143
143
|
/>
|
|
144
|
+
<ValueField
|
|
145
|
+
id="plugin-config-playwright-challenge-wait"
|
|
146
|
+
label={t('challengeWaitMs')}
|
|
147
|
+
hint={t('challengeWaitMsHint')}
|
|
148
|
+
placeholder={t('challengeWaitMsPlaceholder')}
|
|
149
|
+
overriddenLabel={t('overridden')}
|
|
150
|
+
resetLabel={t('reset')}
|
|
151
|
+
invalidLabel={t('invalidText')}
|
|
152
|
+
disabled={disabled}
|
|
153
|
+
{...state.challengeWaitMs}
|
|
154
|
+
onEdit={(text) => { props.edit('challengeWaitMs', text) }}
|
|
155
|
+
onReset={() => { props.resetField('challengeWaitMs') }}
|
|
156
|
+
/>
|
|
144
157
|
</PluginCard>
|
|
145
158
|
)
|
|
146
159
|
}
|
package/src/client/controller.ts
CHANGED
|
@@ -34,6 +34,11 @@ export interface PlaywrightSettings {
|
|
|
34
34
|
denoise?: boolean
|
|
35
35
|
/** How many fetches may render at once (1–200); blank = backend default. */
|
|
36
36
|
maxConcurrency?: number
|
|
37
|
+
/**
|
|
38
|
+
* Bounded wait (ms) for a Cloudflare challenge to clear naturally in the
|
|
39
|
+
* same tab (0–60000); 0 = off (return the first response as-is).
|
|
40
|
+
*/
|
|
41
|
+
challengeWaitMs?: number
|
|
37
42
|
}
|
|
38
43
|
|
|
39
44
|
/** What the Playwright card renders. */
|
|
@@ -50,6 +55,8 @@ export interface PlaywrightCardState extends CardShell {
|
|
|
50
55
|
denoise: CardFieldState
|
|
51
56
|
/** Concurrency input (draft decimal integer). */
|
|
52
57
|
maxConcurrency: CardFieldState
|
|
58
|
+
/** Challenge wait input (draft decimal integer of milliseconds). */
|
|
59
|
+
challengeWaitMs: CardFieldState
|
|
53
60
|
}
|
|
54
61
|
|
|
55
62
|
/** The registration-side face the card's slot entry injects. */
|
|
@@ -78,6 +85,7 @@ export class PlaywrightCardController {
|
|
|
78
85
|
checkboxField('shareBrowserContext'),
|
|
79
86
|
checkboxField('denoise'),
|
|
80
87
|
numberField('maxConcurrency', 1, 200),
|
|
88
|
+
numberField('challengeWaitMs', 0, 60_000),
|
|
81
89
|
],
|
|
82
90
|
)
|
|
83
91
|
this.store = this.form.bind(() => this.projection())
|
|
@@ -92,6 +100,7 @@ export class PlaywrightCardController {
|
|
|
92
100
|
shareBrowserContext: this.form.field('shareBrowserContext'),
|
|
93
101
|
denoise: this.form.field('denoise'),
|
|
94
102
|
maxConcurrency: this.form.field('maxConcurrency'),
|
|
103
|
+
challengeWaitMs: this.form.field('challengeWaitMs'),
|
|
95
104
|
}
|
|
96
105
|
}
|
|
97
106
|
|
package/src/client/locales.ts
CHANGED
|
@@ -14,6 +14,7 @@ export type PlaywrightCardLocaleKey =
|
|
|
14
14
|
| 'shareBrowserContext' | 'shareBrowserContextHint'
|
|
15
15
|
| 'denoise' | 'denoiseHint'
|
|
16
16
|
| 'maxConcurrency' | 'maxConcurrencyHint' | 'maxConcurrencyPlaceholder'
|
|
17
|
+
| 'challengeWaitMs' | 'challengeWaitMsHint' | 'challengeWaitMsPlaceholder'
|
|
17
18
|
| 'overridden' | 'reset' | 'readOnly' | 'expand' | 'collapse'
|
|
18
19
|
| 'save' | 'saving' | 'discard' | 'unsaved' | 'saveFailed' | 'invalidText'
|
|
19
20
|
|
|
@@ -45,6 +46,9 @@ export const en: Record<PlaywrightCardLocaleKey, string> = {
|
|
|
45
46
|
maxConcurrency: 'Max concurrent fetches',
|
|
46
47
|
maxConcurrencyHint: 'How many pages may render at once (1–200). Blank = auto: 4 local browsers, 50 CDP tabs in the remote browser.',
|
|
47
48
|
maxConcurrencyPlaceholder: '(auto: local 4 / CDP 50)',
|
|
49
|
+
challengeWaitMs: 'Cloudflare challenge wait (ms)',
|
|
50
|
+
challengeWaitMsHint: 'Bounded wait for a Cloudflare challenge to clear naturally in the same tab (0–60000; 0 = off). 0 disables: the first response is returned as-is.',
|
|
51
|
+
challengeWaitMsPlaceholder: '(default: 15000)',
|
|
48
52
|
overridden: 'Overridden',
|
|
49
53
|
reset: 'Reset to default',
|
|
50
54
|
readOnly: 'This deployment stores settings read-only.',
|
|
@@ -79,6 +83,9 @@ export const zh: Record<PlaywrightCardLocaleKey, string> = {
|
|
|
79
83
|
maxConcurrency: '最大并发抓取数',
|
|
80
84
|
maxConcurrencyHint: '同时渲染的页面上限(1–200);留空自动:本地 4 个浏览器,CDP 在远端浏览器里 50 个标签页。',
|
|
81
85
|
maxConcurrencyPlaceholder: '(自动:本地 4 / CDP 50)',
|
|
86
|
+
challengeWaitMs: 'Cloudflare 挑战等待上限(毫秒)',
|
|
87
|
+
challengeWaitMsHint: '在同一标签页内有界等待 Cloudflare 验证自然通过(0–60000;0 = 关闭)。关闭时直接返回首次响应——旧版行为。',
|
|
88
|
+
challengeWaitMsPlaceholder: '(默认:15000)',
|
|
82
89
|
overridden: '已覆盖',
|
|
83
90
|
reset: '恢复默认',
|
|
84
91
|
readOnly: '本部署的设置为只读。',
|
package/src/config.ts
CHANGED
|
@@ -30,6 +30,23 @@ export const DEFAULT_MAX_CONCURRENCY_CDP = 50
|
|
|
30
30
|
/** Ceiling the schema accepts for `maxConcurrency` (local slots are browsers). */
|
|
31
31
|
export const MAX_CONCURRENCY_CEILING = 200
|
|
32
32
|
|
|
33
|
+
/**
|
|
34
|
+
* Default bounded wait (ms) for a Cloudflare challenge to clear naturally —
|
|
35
|
+
* the user's real browser passes the verification on its own while the fetch
|
|
36
|
+
* holds the same page and context. Sized to fit the 45s per-fetch deadline
|
|
37
|
+
* with the settle/decode tail (and one retry) still inside it.
|
|
38
|
+
*/
|
|
39
|
+
export const DEFAULT_CHALLENGE_WAIT_MS = 15_000
|
|
40
|
+
|
|
41
|
+
/** Ceiling the schema accepts for `challengeWaitMs`. */
|
|
42
|
+
export const MAX_CHALLENGE_WAIT_MS = 60_000
|
|
43
|
+
|
|
44
|
+
/** Default same-page re-navigation attempts after a challenge wait runs out. */
|
|
45
|
+
export const DEFAULT_CHALLENGE_RETRIES = 1
|
|
46
|
+
|
|
47
|
+
/** Ceiling the schema accepts for `challengeRetries`. */
|
|
48
|
+
export const MAX_CHALLENGE_RETRIES = 3
|
|
49
|
+
|
|
33
50
|
/** Which browser backend serves a fetch. */
|
|
34
51
|
export type PlaywrightBackend = 'local' | 'cdp'
|
|
35
52
|
|
|
@@ -61,6 +78,19 @@ export interface Config {
|
|
|
61
78
|
shareBrowserContext?: boolean
|
|
62
79
|
/** Whether the Readability + DOMPurify denoise pipeline runs before markdown. */
|
|
63
80
|
denoise?: boolean
|
|
81
|
+
/**
|
|
82
|
+
* Bounded wait (ms) for a Cloudflare challenge to clear naturally inside
|
|
83
|
+
* the same page/context. `0` (or any config leaving this at 0) restores the
|
|
84
|
+
* legacy behavior: the first response is final, no waiting — the A/B
|
|
85
|
+
* baseline. Schema default: {@link DEFAULT_CHALLENGE_WAIT_MS}.
|
|
86
|
+
*/
|
|
87
|
+
challengeWaitMs?: number
|
|
88
|
+
/**
|
|
89
|
+
* Same-page re-navigation attempts after a challenge wait window runs out
|
|
90
|
+
* (any partial clearance cookies stay in the context for the retry).
|
|
91
|
+
* Schema default: {@link DEFAULT_CHALLENGE_RETRIES}.
|
|
92
|
+
*/
|
|
93
|
+
challengeRetries?: number
|
|
64
94
|
/**
|
|
65
95
|
* How many fetches may render at once. Blank = backend default (4 for
|
|
66
96
|
* local — each slot launches a browser; 50 for CDP — each slot is a tab in
|
|
@@ -81,6 +111,9 @@ export const Config: z<Config> = z.object({
|
|
|
81
111
|
// Optional on purpose: the effective default depends on `backend`, which a
|
|
82
112
|
// static schema default cannot express.
|
|
83
113
|
maxConcurrency: z.number().step(1).min(1).max(MAX_CONCURRENCY_CEILING),
|
|
114
|
+
// The bounded natural-wait knobs; 0 disables the whole challenge path.
|
|
115
|
+
challengeWaitMs: z.number().step(100).min(0).max(MAX_CHALLENGE_WAIT_MS).default(DEFAULT_CHALLENGE_WAIT_MS),
|
|
116
|
+
challengeRetries: z.number().step(1).min(0).max(MAX_CHALLENGE_RETRIES).default(DEFAULT_CHALLENGE_RETRIES),
|
|
84
117
|
})
|
|
85
118
|
|
|
86
119
|
/**
|
|
@@ -103,6 +136,30 @@ export function effectiveMaxConcurrency(config: Pick<Config, 'backend' | 'maxCon
|
|
|
103
136
|
return config.backend === 'cdp' ? DEFAULT_MAX_CONCURRENCY_CDP : DEFAULT_MAX_CONCURRENCY_LOCAL
|
|
104
137
|
}
|
|
105
138
|
|
|
139
|
+
/**
|
|
140
|
+
* The challenge wait budget a fetch actually runs with: an explicit setting
|
|
141
|
+
* wins, else the schema default. `0` means the challenge path is off.
|
|
142
|
+
*
|
|
143
|
+
* @param config - the resolved settings section (or any partial of it).
|
|
144
|
+
* @returns the bounded wait in milliseconds.
|
|
145
|
+
*/
|
|
146
|
+
export function effectiveChallengeWaitMs(config: Pick<Config, 'challengeWaitMs'>): number {
|
|
147
|
+
if (typeof config.challengeWaitMs === 'number') return config.challengeWaitMs
|
|
148
|
+
return DEFAULT_CHALLENGE_WAIT_MS
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* The same-page retry count a challenge fetch actually runs with: an explicit
|
|
153
|
+
* setting wins, else the schema default.
|
|
154
|
+
*
|
|
155
|
+
* @param config - the resolved settings section (or any partial of it).
|
|
156
|
+
* @returns retries after a wait window runs out.
|
|
157
|
+
*/
|
|
158
|
+
export function effectiveChallengeRetries(config: Pick<Config, 'challengeRetries'>): number {
|
|
159
|
+
if (typeof config.challengeRetries === 'number') return config.challengeRetries
|
|
160
|
+
return DEFAULT_CHALLENGE_RETRIES
|
|
161
|
+
}
|
|
162
|
+
|
|
106
163
|
/**
|
|
107
164
|
* The context mode a fetch actually runs with: profile (a tab in the remote
|
|
108
165
|
* browser's real profile) requires the CDP backend — a local launch has no
|
package/src/index.ts
CHANGED
|
@@ -17,11 +17,13 @@ import { Config } from './config.ts'
|
|
|
17
17
|
import type { ResolvedConfig } from './config.ts'
|
|
18
18
|
import { PlaywrightFetchProvider } from './provider.ts'
|
|
19
19
|
|
|
20
|
-
export { Config, DEFAULT_CDP_ENDPOINT, effectiveContextMode, normalizeCdpEndpoint } from './config.ts'
|
|
20
|
+
export { Config, DEFAULT_CDP_ENDPOINT, DEFAULT_CHALLENGE_RETRIES, DEFAULT_CHALLENGE_WAIT_MS, effectiveChallengeRetries, effectiveChallengeWaitMs, effectiveContextMode, normalizeCdpEndpoint } from './config.ts'
|
|
21
21
|
export type { Config as PlaywrightFetchConfig, CdpContextMode, PlaywrightBackend, ResolvedConfig } from './config.ts'
|
|
22
22
|
export { CdpConnectionPool } from './cdp-pool.ts'
|
|
23
23
|
export type { CdpAcquireMode, CdpConnect, CdpLease } from './cdp-pool.ts'
|
|
24
|
-
export {
|
|
24
|
+
export { CHALLENGE_DOM_PROBE, CHALLENGE_FINISH_RESERVE_MS, CHALLENGE_POLL_INTERVAL_MS, CHALLENGE_TITLE_RE, classifyChallengeHtml, classifyChallengeResponse, isChallengeCompatibleResponse } from './challenge.ts'
|
|
25
|
+
export type { ChallengeVerdict } from './challenge.ts'
|
|
26
|
+
export { PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider, WEB_FETCH_CHALLENGE_CODE } from './provider.ts'
|
|
25
27
|
export { htmlToMarkdown } from './markdown.ts'
|
|
26
28
|
export type { DenoiseMode, DenoiseResult } from './markdown.ts'
|
|
27
29
|
|