dsh-web-fetch-playwright 0.2.3 → 0.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -2
- package/README.zh-CN.md +18 -2
- package/dsh.plugin.json +1 -1
- package/lib/client.js +52 -27
- package/lib/index.js +451 -15
- package/lib/types/challenge.d.ts +83 -0
- package/lib/types/client/controller.d.ts +7 -0
- package/lib/types/client/locales.d.ts +1 -1
- package/lib/types/config.d.ts +42 -0
- package/lib/types/index.d.ts +4 -2
- package/lib/types/markdown.d.ts +2 -0
- package/lib/types/provider.d.ts +44 -1
- package/lib/types/types.d.ts +29 -0
- package/package.json +9 -7
- package/src/challenge.ts +216 -0
- package/src/client/card.tsx +13 -0
- package/src/client/controller.ts +9 -0
- package/src/client/locales.ts +7 -0
- package/src/config.ts +57 -0
- package/src/index.ts +4 -2
- package/src/markdown.ts +55 -0
- package/src/provider.ts +223 -10
- package/src/types.ts +30 -0
package/lib/index.js
CHANGED
|
@@ -21,13 +21,26 @@ import { pathToFileURL } from "node:url";
|
|
|
21
21
|
*/
|
|
22
22
|
/** Default CDP endpoint when the settings section leaves it blank. */
|
|
23
23
|
const DEFAULT_CDP_ENDPOINT = "127.0.0.1:9222";
|
|
24
|
+
/**
|
|
25
|
+
* Default bounded wait (ms) for a Cloudflare challenge to clear naturally —
|
|
26
|
+
* the user's real browser passes the verification on its own while the fetch
|
|
27
|
+
* holds the same page and context. Sized to fit the 45s per-fetch deadline
|
|
28
|
+
* with the settle/decode tail (and one retry) still inside it.
|
|
29
|
+
*/
|
|
30
|
+
const DEFAULT_CHALLENGE_WAIT_MS = 15e3;
|
|
31
|
+
/** Ceiling the schema accepts for `challengeWaitMs`. */
|
|
32
|
+
const MAX_CHALLENGE_WAIT_MS = 6e4;
|
|
33
|
+
/** Default same-page re-navigation attempts after a challenge wait runs out. */
|
|
34
|
+
const DEFAULT_CHALLENGE_RETRIES = 1;
|
|
24
35
|
const Config = z.object({
|
|
25
36
|
backend: z.union([z.const("local"), z.const("cdp")]).default("local"),
|
|
26
37
|
playwrightPath: z.string().default(""),
|
|
27
38
|
cdpEndpoint: z.string().default(""),
|
|
28
39
|
shareBrowserContext: z.boolean().default(true),
|
|
29
40
|
denoise: z.boolean().default(true),
|
|
30
|
-
maxConcurrency: z.number().step(1).min(1).max(200)
|
|
41
|
+
maxConcurrency: z.number().step(1).min(1).max(200),
|
|
42
|
+
challengeWaitMs: z.number().step(100).min(0).max(MAX_CHALLENGE_WAIT_MS).default(DEFAULT_CHALLENGE_WAIT_MS),
|
|
43
|
+
challengeRetries: z.number().step(1).min(0).max(3).default(1)
|
|
31
44
|
});
|
|
32
45
|
/**
|
|
33
46
|
* The concurrency limit a fetch actually runs with: an explicit setting
|
|
@@ -42,6 +55,28 @@ function effectiveMaxConcurrency(config) {
|
|
|
42
55
|
return config.backend === "cdp" ? 50 : 4;
|
|
43
56
|
}
|
|
44
57
|
/**
|
|
58
|
+
* The challenge wait budget a fetch actually runs with: an explicit setting
|
|
59
|
+
* wins, else the schema default. `0` means the challenge path is off.
|
|
60
|
+
*
|
|
61
|
+
* @param config - the resolved settings section (or any partial of it).
|
|
62
|
+
* @returns the bounded wait in milliseconds.
|
|
63
|
+
*/
|
|
64
|
+
function effectiveChallengeWaitMs(config) {
|
|
65
|
+
if (typeof config.challengeWaitMs === "number") return config.challengeWaitMs;
|
|
66
|
+
return DEFAULT_CHALLENGE_WAIT_MS;
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* The same-page retry count a challenge fetch actually runs with: an explicit
|
|
70
|
+
* setting wins, else the schema default.
|
|
71
|
+
*
|
|
72
|
+
* @param config - the resolved settings section (or any partial of it).
|
|
73
|
+
* @returns retries after a wait window runs out.
|
|
74
|
+
*/
|
|
75
|
+
function effectiveChallengeRetries(config) {
|
|
76
|
+
if (typeof config.challengeRetries === "number") return config.challengeRetries;
|
|
77
|
+
return 1;
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
45
80
|
* The context mode a fetch actually runs with: profile (a tab in the remote
|
|
46
81
|
* browser's real profile) requires the CDP backend — a local launch has no
|
|
47
82
|
* meaningful shared profile — and the checkbox (which defaults on, so an
|
|
@@ -73,6 +108,206 @@ function normalizeCdpEndpoint(input) {
|
|
|
73
108
|
return candidate;
|
|
74
109
|
}
|
|
75
110
|
//#endregion
|
|
111
|
+
//#region src/challenge.ts
|
|
112
|
+
/**
|
|
113
|
+
* Cloudflare challenge detection: the pure predicates and in-page probe the
|
|
114
|
+
* provider's bounded natural-wait uses. Nothing here talks to a browser,
|
|
115
|
+
* a clock, or the network — every function is a mapping over a response's
|
|
116
|
+
* status/headers or an HTML string, so the whole surface is unit-testable.
|
|
117
|
+
*
|
|
118
|
+
* Detection is layered, strongest signal first:
|
|
119
|
+
*
|
|
120
|
+
* 1. Response-level — Cloudflare documents that EVERY challenge page type
|
|
121
|
+
* (JS challenge, managed challenge, Turnstile interstitial) carries
|
|
122
|
+
* `cf-mitigated: challenge`. That header is the primary verdict; a
|
|
123
|
+
* 403/503 from a `server: cloudflare` HTML response is the fallback for
|
|
124
|
+
* origins that strip it.
|
|
125
|
+
* 2. Content-level — the interstitial's localized `<title>` family
|
|
126
|
+
* ("Just a moment...", "请稍候…", ...) plus structural markers
|
|
127
|
+
* (`/cdn-cgi/challenge-platform/` scripts, `#challenge-*` elements,
|
|
128
|
+
* `cf-chl-widget-` Turnstile frames, `window._cf_chl_opt`). Markers are
|
|
129
|
+
* deliberately structural: a blog post that merely MENTIONS Cloudflare
|
|
130
|
+
* must not classify as a challenge.
|
|
131
|
+
*
|
|
132
|
+
* The security stance (issue #2): this module only DETECTS and classifies.
|
|
133
|
+
* Solving, spoofing, CAPTCHA answering, and cookie lifting live outside the
|
|
134
|
+
* plugin entirely.
|
|
135
|
+
*
|
|
136
|
+
* @module dsh-web-fetch-playwright/challenge
|
|
137
|
+
*/
|
|
138
|
+
/** How often the bounded wait probes the live page for challenge markers. */
|
|
139
|
+
const CHALLENGE_POLL_INTERVAL_MS = 500;
|
|
140
|
+
/**
|
|
141
|
+
* Tail of the fetch budget held back from the challenge wait so a cleared
|
|
142
|
+
* challenge still has time for its settle (5s networkidle cap) plus the
|
|
143
|
+
* content read and the denoise pipeline.
|
|
144
|
+
*/
|
|
145
|
+
const CHALLENGE_FINISH_RESERVE_MS = 8e3;
|
|
146
|
+
/**
|
|
147
|
+
* The interstitial's localized `<title>` cores, anchored full-match with
|
|
148
|
+
* optional trailing ellipsis/punctuation. Cf. ships dozens of locales; this
|
|
149
|
+
* set covers the major ones — the response header remains the primary
|
|
150
|
+
* signal, so the list only needs to catch header-stripping edges.
|
|
151
|
+
*/
|
|
152
|
+
const CHALLENGE_TITLE_CORES = [
|
|
153
|
+
"just a moment",
|
|
154
|
+
"einen moment",
|
|
155
|
+
"un instant",
|
|
156
|
+
"un momento",
|
|
157
|
+
"um momento",
|
|
158
|
+
"een ogenblik geduld",
|
|
159
|
+
"минуточку",
|
|
160
|
+
"минутку",
|
|
161
|
+
"пожалуйста, подождите",
|
|
162
|
+
"хвилинку",
|
|
163
|
+
"chwileczkę",
|
|
164
|
+
"chvilku strpení",
|
|
165
|
+
"bir dakika",
|
|
166
|
+
"o clipă",
|
|
167
|
+
"μια στιγμή",
|
|
168
|
+
"少し時間がかかります",
|
|
169
|
+
"잠시만요",
|
|
170
|
+
"잠시 기다려주세요",
|
|
171
|
+
"请稍候",
|
|
172
|
+
"請稍候",
|
|
173
|
+
"เดี๋ยวก่อน",
|
|
174
|
+
"vui lòng chờ",
|
|
175
|
+
"chờ một chút",
|
|
176
|
+
"tunggu sebentar",
|
|
177
|
+
"لحظة من فضلك",
|
|
178
|
+
"الرجاء الانتظار",
|
|
179
|
+
"רק רגע",
|
|
180
|
+
"ett ögonblick",
|
|
181
|
+
"et øjeblik",
|
|
182
|
+
"et øyeblikk",
|
|
183
|
+
"hetkinen",
|
|
184
|
+
"pikainen hetki",
|
|
185
|
+
"egy pillanat",
|
|
186
|
+
"моля, изчакайте"
|
|
187
|
+
];
|
|
188
|
+
/** Escape regex metacharacters in a literal title core. */
|
|
189
|
+
function escapeRegExp(text) {
|
|
190
|
+
return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
191
|
+
}
|
|
192
|
+
/**
|
|
193
|
+
* Anchored, case-insensitive match for the localized interstitial title:
|
|
194
|
+
* one of the cores, then only ellipsis/dot/punctuation noise to the end.
|
|
195
|
+
*/
|
|
196
|
+
const CHALLENGE_TITLE_RE = new RegExp(`^(?:${CHALLENGE_TITLE_CORES.map(escapeRegExp).join("|")})[\\s.!??…。!]*$`, "i");
|
|
197
|
+
/**
|
|
198
|
+
* Structural markers a challenge document carries in its HTML. Kept in
|
|
199
|
+
* attribute/assignment form (`id="…"`, `window._cf_chl_opt`) rather than bare
|
|
200
|
+
* substrings so an article that merely QUOTES these strings in prose or code
|
|
201
|
+
* samples does not classify — the marker must look like the real element.
|
|
202
|
+
*/
|
|
203
|
+
const CHALLENGE_HTML_MARKERS = [
|
|
204
|
+
"/cdn-cgi/challenge-platform/",
|
|
205
|
+
"/cdn-cgi/scripts/jsd/main.js",
|
|
206
|
+
"id=\"cf-chl-widget-",
|
|
207
|
+
"window._cf_chl_opt",
|
|
208
|
+
"id=\"challenge-form\"",
|
|
209
|
+
"id=\"challenge-running\"",
|
|
210
|
+
"id=\"challenge-stage\"",
|
|
211
|
+
"id=\"challenge-error-text\"",
|
|
212
|
+
"name=\"cf-challenge\"",
|
|
213
|
+
"checking your browser before accessing"
|
|
214
|
+
];
|
|
215
|
+
/** Title cores of the hard-block page ("Attention Required! | Cloudflare"). */
|
|
216
|
+
const BLOCKED_TITLE_RE = /^attention required!/i;
|
|
217
|
+
/**
|
|
218
|
+
* The hard-block page's body phrase, gated by the structural markers below:
|
|
219
|
+
* the phrase alone must not fail a normal article that happens to contain
|
|
220
|
+
* it in prose.
|
|
221
|
+
*/
|
|
222
|
+
const BLOCKED_BODY_PHRASE = "you have been blocked";
|
|
223
|
+
/** Structural markers of the hard-block page's classic layout. */
|
|
224
|
+
const BLOCKED_BODY_MARKERS = [
|
|
225
|
+
"cf-headline",
|
|
226
|
+
"cf-error-details",
|
|
227
|
+
"cf-block-details"
|
|
228
|
+
];
|
|
229
|
+
/** Extract and trim a document's `<title>` text, '' when absent. */
|
|
230
|
+
function titleOf(html) {
|
|
231
|
+
return (/<title[^>]*>([^<]*)<\/title>/i.exec(html)?.[1] ?? "").trim();
|
|
232
|
+
}
|
|
233
|
+
/** Whether a content-type mime is an HTML document (challenge pages are). */
|
|
234
|
+
function isHtmlMime(contentType) {
|
|
235
|
+
const mime = contentType.replace(/;.*$/s, "").trim().toLowerCase();
|
|
236
|
+
return mime === "" || mime === "text/html" || mime === "application/xhtml+xml";
|
|
237
|
+
}
|
|
238
|
+
/**
|
|
239
|
+
* Classify a navigation response from its status and headers.
|
|
240
|
+
*
|
|
241
|
+
* @param status - the response's HTTP status code.
|
|
242
|
+
* @param headers - the response's headers (case as Playwright serves: lowercased).
|
|
243
|
+
* @returns `'challenge'` when the response is a Cloudflare challenge, else `'none'`.
|
|
244
|
+
*/
|
|
245
|
+
function classifyChallengeResponse(status, headers) {
|
|
246
|
+
const header = (name) => (headers[name] ?? "").trim().toLowerCase();
|
|
247
|
+
if (header("cf-mitigated") === "challenge") return "challenge";
|
|
248
|
+
if ((status === 403 || status === 503) && header("server").includes("cloudflare") && isHtmlMime(header("content-type"))) return "challenge";
|
|
249
|
+
return "none";
|
|
250
|
+
}
|
|
251
|
+
/**
|
|
252
|
+
* Whether a response is even a candidate for the content-level checks —
|
|
253
|
+
* the suspicion gate. Cloudflare interstitials NEVER ship a plain 200 from
|
|
254
|
+
* a non-Cloudflare origin: they are 403/429/503, or served from a Cloudflare
|
|
255
|
+
* edge (`server: cloudflare`, `cf-ray`). Running the title/marker fallback
|
|
256
|
+
* only on such responses means a normal article — whatever it quotes about
|
|
257
|
+
* Cloudflare, in any language — can never be misread as a challenge. This is
|
|
258
|
+
* the tiering the industry converges on: protocol signals first, content
|
|
259
|
+
* markers only as a gated fallback.
|
|
260
|
+
*
|
|
261
|
+
* @param status - the response's HTTP status code.
|
|
262
|
+
* @param headers - the response's headers (lowercased, as Playwright serves).
|
|
263
|
+
* @returns true when the content-level fallback may run.
|
|
264
|
+
*/
|
|
265
|
+
function isChallengeCompatibleResponse(status, headers) {
|
|
266
|
+
if (status === 403 || status === 429 || status === 503) return true;
|
|
267
|
+
const header = (name) => (headers[name] ?? "").trim().toLowerCase();
|
|
268
|
+
return header("server").includes("cloudflare") || header("cf-ray") !== "";
|
|
269
|
+
}
|
|
270
|
+
/**
|
|
271
|
+
* Classify a rendered document from its HTML: the localized interstitial
|
|
272
|
+
* title family and the structural markers above say `challenge`; the
|
|
273
|
+
* hard-block page says `blocked` (waiting cannot clear it — the caller
|
|
274
|
+
* should fail immediately instead of burning its budget).
|
|
275
|
+
*
|
|
276
|
+
* @param html - the document HTML as `page.content()` returns it.
|
|
277
|
+
* @returns the verdict for this HTML.
|
|
278
|
+
*/
|
|
279
|
+
function classifyChallengeHtml(html) {
|
|
280
|
+
if (html.length === 0) return "none";
|
|
281
|
+
const title = titleOf(html);
|
|
282
|
+
if (BLOCKED_TITLE_RE.test(title)) return "blocked";
|
|
283
|
+
const lowered = html.toLowerCase();
|
|
284
|
+
if (lowered.includes(BLOCKED_BODY_PHRASE) && BLOCKED_BODY_MARKERS.some((marker) => lowered.includes(marker))) return "blocked";
|
|
285
|
+
if (CHALLENGE_TITLE_RE.test(title)) return "challenge";
|
|
286
|
+
if (CHALLENGE_HTML_MARKERS.some((marker) => lowered.includes(marker))) return "challenge";
|
|
287
|
+
if (lowered.includes("class=\"ray-id\"") && lowered.includes("class=\"footer-inner\"")) return "challenge";
|
|
288
|
+
return "none";
|
|
289
|
+
}
|
|
290
|
+
/**
|
|
291
|
+
* The in-page probe the bounded wait runs against the LIVE document (via
|
|
292
|
+
* `page.evaluate`): true while challenge markers are still present. This is
|
|
293
|
+
* what makes SPA-style clears visible — the document swaps its content
|
|
294
|
+
* without any navigation, so only a DOM read can see the change. The title
|
|
295
|
+
* regex is baked in at build time from the same single source the Node-side
|
|
296
|
+
* check uses, so the two can never drift apart.
|
|
297
|
+
*/
|
|
298
|
+
const CHALLENGE_DOM_PROBE = `(() => {
|
|
299
|
+
const re = new RegExp(${JSON.stringify(CHALLENGE_TITLE_RE.source)}, ${JSON.stringify(CHALLENGE_TITLE_RE.flags)})
|
|
300
|
+
if (re.test(document.title.trim())) return true
|
|
301
|
+
if (document.querySelector('#challenge-form, #challenge-running, #challenge-stage, #challenge-error-text')) return true
|
|
302
|
+
if (document.querySelector('[id^="cf-chl-widget-"]')) return true
|
|
303
|
+
for (const script of Array.from(document.scripts)) {
|
|
304
|
+
const src = script.getAttribute('src') || ''
|
|
305
|
+
if (src.includes('/cdn-cgi/challenge-platform/') || src.includes('/cdn-cgi/scripts/jsd/main.js')) return true
|
|
306
|
+
}
|
|
307
|
+
if (document.querySelector('.footer .footer-inner .ray-id')) return true
|
|
308
|
+
return false
|
|
309
|
+
})()`;
|
|
310
|
+
//#endregion
|
|
76
311
|
//#region src/cdp-pool.ts
|
|
77
312
|
/** A reusable `connectOverCDP` session handing out per-fetch pages. */
|
|
78
313
|
var CdpConnectionPool = class {
|
|
@@ -232,6 +467,8 @@ var CdpConnectionPool = class {
|
|
|
232
467
|
* The denoise pipeline: rendered HTML → sanitized article HTML → markdown.
|
|
233
468
|
*
|
|
234
469
|
* The classic stack, in order: jsdom parses the page Playwright rendered;
|
|
470
|
+
* inline `data:` image payloads are elided to size placeholders (build
|
|
471
|
+
* tools inline images as base64, which would otherwise dominate the body);
|
|
235
472
|
* Mozilla Readability extracts the article (dropping nav bars, sidebars,
|
|
236
473
|
* footers, and ad chrome by scoring link density and text mass); DOMPurify
|
|
237
474
|
* sanitizes whatever HTML remains and forbids the layout tags noise lives in;
|
|
@@ -273,6 +510,46 @@ const FORBID_ATTR = [
|
|
|
273
510
|
"aria-hidden",
|
|
274
511
|
"role"
|
|
275
512
|
];
|
|
513
|
+
/**
|
|
514
|
+
* Elide inline `data:` image payloads to `data:<mime>;base64,...<size>`.
|
|
515
|
+
*
|
|
516
|
+
* Build tools (Docusaurus/webpack, Hugo) inline images above a size cutoff
|
|
517
|
+
* straight into the HTML as data URIs — measured on an onlyoffice.com docs
|
|
518
|
+
* page, 12 inline PNGs were 21.6% of the HTML and, surviving Readability
|
|
519
|
+
* (they are content), DOMPurify (img+data: is on its DATA_URI_TAGS
|
|
520
|
+
* allowlist), and Turndown's default image rule, ended up **65% of the
|
|
521
|
+
* returned markdown body**. Network-level filtering cannot touch them: a
|
|
522
|
+
* data URI is never fetched, so the provider's subrequest abort misses it.
|
|
523
|
+
* The placeholder keeps alt text, MIME type, and the approximate size, so
|
|
524
|
+
* the model still knows an inline image existed. The marker is pure ASCII
|
|
525
|
+
* because Readability re-resolves image srcs through `new URL()`, which
|
|
526
|
+
* would percent-encode anything else.
|
|
527
|
+
*/
|
|
528
|
+
function elideDataUriImages(document) {
|
|
529
|
+
for (const img of Array.from(document.querySelectorAll("img"))) {
|
|
530
|
+
const src = img.getAttribute("src");
|
|
531
|
+
if (src === null || !src.startsWith("data:")) continue;
|
|
532
|
+
img.setAttribute("src", dataUriPlaceholder(src));
|
|
533
|
+
}
|
|
534
|
+
}
|
|
535
|
+
/**
|
|
536
|
+
* Shorten one data URI to its header plus a size marker.
|
|
537
|
+
* @param src - the original `data:` URI.
|
|
538
|
+
* @returns e.g. `data:image/png;base64,...8.9KB` (base64 sizes are decoded
|
|
539
|
+
* bytes; non-base64 payloads report their character count).
|
|
540
|
+
*/
|
|
541
|
+
function dataUriPlaceholder(src) {
|
|
542
|
+
const commaIndex = src.indexOf(",");
|
|
543
|
+
const header = commaIndex === -1 ? src : src.slice(0, commaIndex + 1);
|
|
544
|
+
const payload = commaIndex === -1 ? "" : src.slice(commaIndex + 1);
|
|
545
|
+
return `${header}...${humanSize(/;base64$/i.test(header.slice(5, -1)) ? Math.max(0, Math.floor(payload.length / 4) * 3 - (payload.match(/=+$/)?.[0].length ?? 0)) : payload.length)}`;
|
|
546
|
+
}
|
|
547
|
+
/** Render a byte count as `123B` / `8.9KB` / `1.2MB`. */
|
|
548
|
+
function humanSize(bytes) {
|
|
549
|
+
if (bytes < 1024) return `${String(bytes)}B`;
|
|
550
|
+
if (bytes < 1048576) return `${(bytes / 1024).toFixed(1)}KB`;
|
|
551
|
+
return `${(bytes / 1048576).toFixed(1)}MB`;
|
|
552
|
+
}
|
|
276
553
|
/** The shared converter: same style options as `dsh-tool-web`'s renderer. */
|
|
277
554
|
const turndown = new TurndownService({
|
|
278
555
|
headingStyle: "atx",
|
|
@@ -339,6 +616,7 @@ function htmlToMarkdown(html, url) {
|
|
|
339
616
|
});
|
|
340
617
|
const purify = createDOMPurify(dom.window);
|
|
341
618
|
const document = dom.window.document;
|
|
619
|
+
elideDataUriImages(document);
|
|
342
620
|
let source = null;
|
|
343
621
|
let title;
|
|
344
622
|
try {
|
|
@@ -573,10 +851,30 @@ function existsAsFile(path) {
|
|
|
573
851
|
* browser can reach. Profile mode additionally acts WITH the remote
|
|
574
852
|
* browser's logged-in sessions (see the README's risk notes).
|
|
575
853
|
*
|
|
854
|
+
* Cloudflare challenges (issue #2): when a navigation lands on a challenge
|
|
855
|
+
* interstitial, the fetch waits — on the SAME page and in the SAME browser
|
|
856
|
+
* context, so the browser's natural verification and any clearance cookies
|
|
857
|
+
* it earns apply — for a bounded, configurable window
|
|
858
|
+
* (`challengeWaitMs`, default 15s; 0 restores the legacy first-response
|
|
859
|
+
* behavior). The wait tracks the LAST main-frame navigation response (the
|
|
860
|
+
* real page reloads in after the challenge clears) and watches the live DOM
|
|
861
|
+
* so SPA-style clears are caught too. It never clicks, never injects
|
|
862
|
+
* answers, never touches cookies itself; when the budget runs out the fetch
|
|
863
|
+
* fails with {@link WEB_FETCH_CHALLENGE_CODE} instead of returning the
|
|
864
|
+
* interstitial as content.
|
|
865
|
+
*
|
|
576
866
|
* @module dsh-web-fetch-playwright/provider
|
|
577
867
|
*/
|
|
578
868
|
/** Stable id this provider registers under (the bundle patch pins it). */
|
|
579
869
|
const PLAYWRIGHT_FETCH_PROVIDER_ID = "playwright";
|
|
870
|
+
/**
|
|
871
|
+
* Error code for a Cloudflare challenge the bounded natural wait could not
|
|
872
|
+
* clear. Provider-specific by design — the web seam's `code` is an open
|
|
873
|
+
* string and consumers must tolerate provider-specific codes — so callers
|
|
874
|
+
* can tell "the site challenged us and the browser did not pass" apart from
|
|
875
|
+
* a transport timeout or a provider bug.
|
|
876
|
+
*/
|
|
877
|
+
const WEB_FETCH_CHALLENGE_CODE = "WEB_FETCH_CHALLENGE";
|
|
580
878
|
/** Maximum accepted request URL length (http-provider parity). */
|
|
581
879
|
const MAX_URL_LENGTH = 2048;
|
|
582
880
|
/** Cap on the decoded markdown/HTML body this provider returns. */
|
|
@@ -848,32 +1146,66 @@ var PlaywrightFetchProvider = class {
|
|
|
848
1146
|
throw new WebError(`cannot launch the local browser (${source}); run \`playwright install chromium\` or point the settings path at a playwright/browser executable. ${String(error instanceof Error ? error.message : error)}`, "WEB_PROVIDER_ERROR", { cause: error });
|
|
849
1147
|
}
|
|
850
1148
|
}
|
|
851
|
-
/**
|
|
1149
|
+
/**
|
|
1150
|
+
* Navigate, settle, and decode one URL inside an open session's tab —
|
|
1151
|
+
* waiting out any Cloudflare interstitial within the bounded challenge
|
|
1152
|
+
* budget before the final document is read (see the class docs).
|
|
1153
|
+
*/
|
|
852
1154
|
async retrieve(session, url, config, deadline) {
|
|
853
1155
|
const page = session.page;
|
|
854
|
-
const
|
|
1156
|
+
const challengeWaitMs = effectiveChallengeWaitMs(config);
|
|
1157
|
+
const tracker = challengeWaitMs > 0 ? trackMainFrameResponses(page) : void 0;
|
|
1158
|
+
let response = await page.goto(url.toString(), {
|
|
855
1159
|
waitUntil: "domcontentloaded",
|
|
856
1160
|
timeout: deadline.remainingMs()
|
|
857
1161
|
});
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
1162
|
+
tracker?.seed(response);
|
|
1163
|
+
let challengeEntryResponse = null;
|
|
1164
|
+
if (challengeWaitMs > 0) {
|
|
1165
|
+
let attemptsLeft = effectiveChallengeRetries(config) + 1;
|
|
1166
|
+
for (;;) {
|
|
1167
|
+
const verdict = await this.verdictAfterLoad(page, tracker?.last() ?? response);
|
|
1168
|
+
if (verdict === "blocked") throw new WebError(`the site hard-blocked this fetch at its Cloudflare edge (waiting cannot clear it): ${page.url()}`, WEB_FETCH_CHALLENGE_CODE);
|
|
1169
|
+
if (verdict !== "challenge") break;
|
|
1170
|
+
challengeEntryResponse = tracker?.last() ?? response;
|
|
1171
|
+
if (await this.waitForChallengeClear(page, deadline, challengeWaitMs)) {
|
|
1172
|
+
let chained = false;
|
|
1173
|
+
try {
|
|
1174
|
+
chained = classifyChallengeHtml(await page.content()) === "challenge";
|
|
1175
|
+
} catch {
|
|
1176
|
+
chained = false;
|
|
1177
|
+
}
|
|
1178
|
+
if (!chained) break;
|
|
1179
|
+
}
|
|
1180
|
+
if (--attemptsLeft <= 0) {
|
|
1181
|
+
const lastStatus = (tracker?.last() ?? response)?.status();
|
|
1182
|
+
throw new WebError(`the site kept serving a Cloudflare challenge (last status ${lastStatus === void 0 ? "unknown" : String(lastStatus)}) for up to ${String(challengeWaitMs)}ms across ${String(effectiveChallengeRetries(config) + 1)} attempt(s); the browser did not clear it naturally — retry later, raise challengeWaitMs, or use a profile whose browser already holds clearance`, WEB_FETCH_CHALLENGE_CODE);
|
|
1183
|
+
}
|
|
1184
|
+
response = await page.goto(url.toString(), {
|
|
1185
|
+
waitUntil: "domcontentloaded",
|
|
1186
|
+
timeout: deadline.remainingMs()
|
|
1187
|
+
});
|
|
1188
|
+
tracker?.seed(response);
|
|
1189
|
+
}
|
|
867
1190
|
}
|
|
1191
|
+
const finalResponse = (challengeWaitMs > 0 ? tracker?.last() : void 0) ?? response;
|
|
1192
|
+
const kind = classifyContentType(finalResponse?.headers()["content-type"]);
|
|
1193
|
+
if (kind === void 0) throw new WebError(`unsupported content type "${finalResponse?.headers()["content-type"] ?? "unknown"}"`, "WEB_UNSUPPORTED_CONTENT_TYPE");
|
|
1194
|
+
const finalUrl = page.url();
|
|
1195
|
+
const statusCode = finalResponse !== null && !(challengeEntryResponse !== null && finalResponse === challengeEntryResponse) ? finalResponse.status() : 200;
|
|
1196
|
+
if (kind === "text") return capResult(finalUrl, statusCode, {
|
|
1197
|
+
kind: "text",
|
|
1198
|
+
content: finalResponse !== null ? await finalResponse.text() : await page.content()
|
|
1199
|
+
});
|
|
868
1200
|
await page.waitForLoadState("networkidle", { timeout: Math.min(SETTLE_MS, deadline.remainingMs()) }).catch(() => {});
|
|
869
1201
|
const html = await page.content();
|
|
870
|
-
if (!config.denoise) return capResult(finalUrl,
|
|
1202
|
+
if (!config.denoise) return capResult(finalUrl, statusCode, {
|
|
871
1203
|
kind: "html",
|
|
872
1204
|
content: html
|
|
873
1205
|
});
|
|
874
1206
|
const bounded = html.length > MAX_PIPELINE_INPUT_CHARS ? html.slice(0, MAX_PIPELINE_INPUT_CHARS) : html;
|
|
875
1207
|
const { markdown } = htmlToMarkdown(bounded, finalUrl);
|
|
876
|
-
const result = capResult(finalUrl,
|
|
1208
|
+
const result = capResult(finalUrl, statusCode, {
|
|
877
1209
|
kind: "text",
|
|
878
1210
|
content: markdown
|
|
879
1211
|
});
|
|
@@ -882,7 +1214,111 @@ var PlaywrightFetchProvider = class {
|
|
|
882
1214
|
truncated: true
|
|
883
1215
|
} : result;
|
|
884
1216
|
}
|
|
1217
|
+
/**
|
|
1218
|
+
* Whether the freshly loaded document is a Cloudflare interstitial: the
|
|
1219
|
+
* response-level signals first (the documented `cf-mitigated` header, then
|
|
1220
|
+
* the 403/503 + cloudflare fallback), then — gated behind
|
|
1221
|
+
* {@link isChallengeCompatibleResponse}, because interstitials never ship
|
|
1222
|
+
* a plain 200 — the localized title family and structural markers, which
|
|
1223
|
+
* also recognize the hard-block page waiting can never clear.
|
|
1224
|
+
*/
|
|
1225
|
+
async verdictAfterLoad(page, response) {
|
|
1226
|
+
if (response !== null && classifyChallengeResponse(response.status(), response.headers()) === "challenge") return "challenge";
|
|
1227
|
+
if (response !== null && !isChallengeCompatibleResponse(response.status(), response.headers())) return "none";
|
|
1228
|
+
if (response === null || classifyContentType(response.headers()["content-type"]) === "html") try {
|
|
1229
|
+
return classifyChallengeHtml(await page.content());
|
|
1230
|
+
} catch {
|
|
1231
|
+
return "none";
|
|
1232
|
+
}
|
|
1233
|
+
return "none";
|
|
1234
|
+
}
|
|
1235
|
+
/**
|
|
1236
|
+
* The bounded natural wait: poll the live document for challenge markers
|
|
1237
|
+
* until they are gone (the user's real browser passed the verification and
|
|
1238
|
+
* reloaded into the real page, or swapped it in SPA-style), the per-fetch
|
|
1239
|
+
* wait budget runs out, or the fetch's deadline aborts. No clicks, no
|
|
1240
|
+
* injected answers — the browser either clears it on its own or it does not.
|
|
1241
|
+
*
|
|
1242
|
+
* @returns true when the challenge cleared; false when the budget ran out.
|
|
1243
|
+
*/
|
|
1244
|
+
async waitForChallengeClear(page, deadline, challengeWaitMs) {
|
|
1245
|
+
const budget = Math.min(challengeWaitMs, deadline.remainingMs() - CHALLENGE_FINISH_RESERVE_MS);
|
|
1246
|
+
if (budget <= 0) return false;
|
|
1247
|
+
const until = Date.now() + budget;
|
|
1248
|
+
for (;;) {
|
|
1249
|
+
if (deadline.signal.aborted) throw new Error("challenge wait aborted");
|
|
1250
|
+
if (await probeStillOnChallenge(page)) {
|
|
1251
|
+
const remaining = until - Date.now();
|
|
1252
|
+
if (remaining <= 0) return false;
|
|
1253
|
+
await sleep(Math.min(500, remaining));
|
|
1254
|
+
continue;
|
|
1255
|
+
}
|
|
1256
|
+
await page.waitForLoadState("domcontentloaded", { timeout: Math.min(deadline.remainingMs(), 2e3) }).catch(() => {});
|
|
1257
|
+
return true;
|
|
1258
|
+
}
|
|
1259
|
+
}
|
|
885
1260
|
};
|
|
1261
|
+
/** Resolve after `ms` — the bounded wait's inter-poll nap. */
|
|
1262
|
+
function sleep(ms) {
|
|
1263
|
+
return new Promise((resolve) => {
|
|
1264
|
+
setTimeout(resolve, ms);
|
|
1265
|
+
});
|
|
1266
|
+
}
|
|
1267
|
+
/**
|
|
1268
|
+
* Whether a response event belongs to the page's main frame as a document
|
|
1269
|
+
* navigation — the filter the challenge wait uses to keep the LAST such
|
|
1270
|
+
* response (challenge pages reload the same URL into the real document).
|
|
1271
|
+
* Structural members are optional; when the backend does not expose them the
|
|
1272
|
+
* check degrades to "looks like a document" so fakes stay usable.
|
|
1273
|
+
*/
|
|
1274
|
+
function isMainFrameDocument(response, page) {
|
|
1275
|
+
const request = response.request?.();
|
|
1276
|
+
if (request === void 0) return true;
|
|
1277
|
+
if (typeof request.isNavigationRequest === "function" && !request.isNavigationRequest()) return false;
|
|
1278
|
+
const resourceType = typeof request.resourceType === "function" ? request.resourceType() : void 0;
|
|
1279
|
+
if (resourceType !== void 0 && resourceType !== "document") return false;
|
|
1280
|
+
if (typeof request.frame === "function" && typeof page.mainFrame === "function") return request.frame() === page.mainFrame();
|
|
1281
|
+
return true;
|
|
1282
|
+
}
|
|
1283
|
+
/**
|
|
1284
|
+
* Watch the page for main-frame navigation responses so a challenge that
|
|
1285
|
+
* reloads into the real document hands the caller THAT response — status,
|
|
1286
|
+
* headers — instead of the challenge's. Best-effort: a page without the
|
|
1287
|
+
* listener falls back to SPA-style content polling only.
|
|
1288
|
+
*/
|
|
1289
|
+
function trackMainFrameResponses(page) {
|
|
1290
|
+
let last = null;
|
|
1291
|
+
try {
|
|
1292
|
+
page.on?.("response", (response) => {
|
|
1293
|
+
if (isMainFrameDocument(response, page)) last = response;
|
|
1294
|
+
});
|
|
1295
|
+
} catch {}
|
|
1296
|
+
return {
|
|
1297
|
+
last: () => last,
|
|
1298
|
+
seed: (response) => {
|
|
1299
|
+
if (response !== null) last = response;
|
|
1300
|
+
}
|
|
1301
|
+
};
|
|
1302
|
+
}
|
|
1303
|
+
/**
|
|
1304
|
+
* Probe the live document for challenge markers. Prefers the in-page probe
|
|
1305
|
+
* (`page.evaluate`) — the only thing that can see SPA-style clears — and
|
|
1306
|
+
* falls back to reading and classifying the serialized HTML when the backend
|
|
1307
|
+
* does not expose evaluate. A probe that throws mid-navigation (execution
|
|
1308
|
+
* context destroyed while the challenge reloads) counts as "still on the
|
|
1309
|
+
* challenge": the next poll sees the fresh document.
|
|
1310
|
+
*/
|
|
1311
|
+
async function probeStillOnChallenge(page) {
|
|
1312
|
+
if (typeof page.evaluate === "function") try {
|
|
1313
|
+
const verdict = await page.evaluate(CHALLENGE_DOM_PROBE);
|
|
1314
|
+
if (typeof verdict === "boolean") return verdict;
|
|
1315
|
+
} catch {}
|
|
1316
|
+
try {
|
|
1317
|
+
return classifyChallengeHtml(await page.content()) !== "none";
|
|
1318
|
+
} catch {
|
|
1319
|
+
return true;
|
|
1320
|
+
}
|
|
1321
|
+
}
|
|
886
1322
|
/** Apply the decoded-body cap and flag the cut. */
|
|
887
1323
|
function capResult(url, statusCode, body) {
|
|
888
1324
|
const truncated = body.content.length > MAX_BODY_CHARS;
|
|
@@ -985,4 +1421,4 @@ function apply(ctx, config) {
|
|
|
985
1421
|
ctx.web.registerFetchProvider(provider);
|
|
986
1422
|
}
|
|
987
1423
|
//#endregion
|
|
988
|
-
export { CdpConnectionPool, Config, DEFAULT_CDP_ENDPOINT, PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider, WEB_FETCH_PLAYWRIGHT_SETTINGS_NAMESPACE, apply, effectiveContextMode, htmlToMarkdown, inject, name, normalizeCdpEndpoint };
|
|
1424
|
+
export { CHALLENGE_DOM_PROBE, CHALLENGE_FINISH_RESERVE_MS, CHALLENGE_POLL_INTERVAL_MS, CHALLENGE_TITLE_RE, CdpConnectionPool, Config, DEFAULT_CDP_ENDPOINT, DEFAULT_CHALLENGE_RETRIES, DEFAULT_CHALLENGE_WAIT_MS, PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider, WEB_FETCH_CHALLENGE_CODE, WEB_FETCH_PLAYWRIGHT_SETTINGS_NAMESPACE, apply, classifyChallengeHtml, classifyChallengeResponse, effectiveChallengeRetries, effectiveChallengeWaitMs, effectiveContextMode, htmlToMarkdown, inject, isChallengeCompatibleResponse, name, normalizeCdpEndpoint };
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cloudflare challenge detection: the pure predicates and in-page probe the
|
|
3
|
+
* provider's bounded natural-wait uses. Nothing here talks to a browser,
|
|
4
|
+
* a clock, or the network — every function is a mapping over a response's
|
|
5
|
+
* status/headers or an HTML string, so the whole surface is unit-testable.
|
|
6
|
+
*
|
|
7
|
+
* Detection is layered, strongest signal first:
|
|
8
|
+
*
|
|
9
|
+
* 1. Response-level — Cloudflare documents that EVERY challenge page type
|
|
10
|
+
* (JS challenge, managed challenge, Turnstile interstitial) carries
|
|
11
|
+
* `cf-mitigated: challenge`. That header is the primary verdict; a
|
|
12
|
+
* 403/503 from a `server: cloudflare` HTML response is the fallback for
|
|
13
|
+
* origins that strip it.
|
|
14
|
+
* 2. Content-level — the interstitial's localized `<title>` family
|
|
15
|
+
* ("Just a moment...", "请稍候…", ...) plus structural markers
|
|
16
|
+
* (`/cdn-cgi/challenge-platform/` scripts, `#challenge-*` elements,
|
|
17
|
+
* `cf-chl-widget-` Turnstile frames, `window._cf_chl_opt`). Markers are
|
|
18
|
+
* deliberately structural: a blog post that merely MENTIONS Cloudflare
|
|
19
|
+
* must not classify as a challenge.
|
|
20
|
+
*
|
|
21
|
+
* The security stance (issue #2): this module only DETECTS and classifies.
|
|
22
|
+
* Solving, spoofing, CAPTCHA answering, and cookie lifting live outside the
|
|
23
|
+
* plugin entirely.
|
|
24
|
+
*
|
|
25
|
+
* @module dsh-web-fetch-playwright/challenge
|
|
26
|
+
*/
|
|
27
|
+
/** How often the bounded wait probes the live page for challenge markers. */
|
|
28
|
+
export declare const CHALLENGE_POLL_INTERVAL_MS = 500;
|
|
29
|
+
/**
|
|
30
|
+
* Tail of the fetch budget held back from the challenge wait so a cleared
|
|
31
|
+
* challenge still has time for its settle (5s networkidle cap) plus the
|
|
32
|
+
* content read and the denoise pipeline.
|
|
33
|
+
*/
|
|
34
|
+
export declare const CHALLENGE_FINISH_RESERVE_MS = 8000;
|
|
35
|
+
/**
|
|
36
|
+
* Anchored, case-insensitive match for the localized interstitial title:
|
|
37
|
+
* one of the cores, then only ellipsis/dot/punctuation noise to the end.
|
|
38
|
+
*/
|
|
39
|
+
export declare const CHALLENGE_TITLE_RE: RegExp;
|
|
40
|
+
/** The verdicts detection can return. */
|
|
41
|
+
export type ChallengeVerdict = 'none' | 'challenge' | 'blocked';
|
|
42
|
+
/**
|
|
43
|
+
* Classify a navigation response from its status and headers.
|
|
44
|
+
*
|
|
45
|
+
* @param status - the response's HTTP status code.
|
|
46
|
+
* @param headers - the response's headers (case as Playwright serves: lowercased).
|
|
47
|
+
* @returns `'challenge'` when the response is a Cloudflare challenge, else `'none'`.
|
|
48
|
+
*/
|
|
49
|
+
export declare function classifyChallengeResponse(status: number, headers: Record<string, string>): 'none' | 'challenge';
|
|
50
|
+
/**
|
|
51
|
+
* Whether a response is even a candidate for the content-level checks —
|
|
52
|
+
* the suspicion gate. Cloudflare interstitials NEVER ship a plain 200 from
|
|
53
|
+
* a non-Cloudflare origin: they are 403/429/503, or served from a Cloudflare
|
|
54
|
+
* edge (`server: cloudflare`, `cf-ray`). Running the title/marker fallback
|
|
55
|
+
* only on such responses means a normal article — whatever it quotes about
|
|
56
|
+
* Cloudflare, in any language — can never be misread as a challenge. This is
|
|
57
|
+
* the tiering the industry converges on: protocol signals first, content
|
|
58
|
+
* markers only as a gated fallback.
|
|
59
|
+
*
|
|
60
|
+
* @param status - the response's HTTP status code.
|
|
61
|
+
* @param headers - the response's headers (lowercased, as Playwright serves).
|
|
62
|
+
* @returns true when the content-level fallback may run.
|
|
63
|
+
*/
|
|
64
|
+
export declare function isChallengeCompatibleResponse(status: number, headers: Record<string, string>): boolean;
|
|
65
|
+
/**
|
|
66
|
+
* Classify a rendered document from its HTML: the localized interstitial
|
|
67
|
+
* title family and the structural markers above say `challenge`; the
|
|
68
|
+
* hard-block page says `blocked` (waiting cannot clear it — the caller
|
|
69
|
+
* should fail immediately instead of burning its budget).
|
|
70
|
+
*
|
|
71
|
+
* @param html - the document HTML as `page.content()` returns it.
|
|
72
|
+
* @returns the verdict for this HTML.
|
|
73
|
+
*/
|
|
74
|
+
export declare function classifyChallengeHtml(html: string): ChallengeVerdict;
|
|
75
|
+
/**
|
|
76
|
+
* The in-page probe the bounded wait runs against the LIVE document (via
|
|
77
|
+
* `page.evaluate`): true while challenge markers are still present. This is
|
|
78
|
+
* what makes SPA-style clears visible — the document swaps its content
|
|
79
|
+
* without any navigation, so only a DOM read can see the change. The title
|
|
80
|
+
* regex is baked in at build time from the same single source the Node-side
|
|
81
|
+
* check uses, so the two can never drift apart.
|
|
82
|
+
*/
|
|
83
|
+
export declare const CHALLENGE_DOM_PROBE: string;
|
|
@@ -30,6 +30,11 @@ export interface PlaywrightSettings {
|
|
|
30
30
|
denoise?: boolean;
|
|
31
31
|
/** How many fetches may render at once (1–200); blank = backend default. */
|
|
32
32
|
maxConcurrency?: number;
|
|
33
|
+
/**
|
|
34
|
+
* Bounded wait (ms) for a Cloudflare challenge to clear naturally in the
|
|
35
|
+
* same tab (0–60000); 0 = off (return the first response as-is).
|
|
36
|
+
*/
|
|
37
|
+
challengeWaitMs?: number;
|
|
33
38
|
}
|
|
34
39
|
/** What the Playwright card renders. */
|
|
35
40
|
export interface PlaywrightCardState extends CardShell {
|
|
@@ -45,6 +50,8 @@ export interface PlaywrightCardState extends CardShell {
|
|
|
45
50
|
denoise: CardFieldState;
|
|
46
51
|
/** Concurrency input (draft decimal integer). */
|
|
47
52
|
maxConcurrency: CardFieldState;
|
|
53
|
+
/** Challenge wait input (draft decimal integer of milliseconds). */
|
|
54
|
+
challengeWaitMs: CardFieldState;
|
|
48
55
|
}
|
|
49
56
|
/** The registration-side face the card's slot entry injects. */
|
|
50
57
|
export interface PlaywrightCardFace extends CardActions {
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* @module dsh-web-fetch-playwright/client/locales
|
|
6
6
|
*/
|
|
7
7
|
/** Locale keys this card renders. */
|
|
8
|
-
export type PlaywrightCardLocaleKey = 'title' | 'description' | 'backendLabel' | 'backendLocal' | 'backendLocalHint' | 'backendCdp' | 'backendCdpHint' | 'playwrightPath' | 'playwrightPathHint' | 'playwrightPathPlaceholder' | 'cdpEndpoint' | 'cdpEndpointHint' | 'shareBrowserContext' | 'shareBrowserContextHint' | 'denoise' | 'denoiseHint' | 'maxConcurrency' | 'maxConcurrencyHint' | 'maxConcurrencyPlaceholder' | 'overridden' | 'reset' | 'readOnly' | 'expand' | 'collapse' | 'save' | 'saving' | 'discard' | 'unsaved' | 'saveFailed' | 'invalidText';
|
|
8
|
+
export type PlaywrightCardLocaleKey = 'title' | 'description' | 'backendLabel' | 'backendLocal' | 'backendLocalHint' | 'backendCdp' | 'backendCdpHint' | 'playwrightPath' | 'playwrightPathHint' | 'playwrightPathPlaceholder' | 'cdpEndpoint' | 'cdpEndpointHint' | 'shareBrowserContext' | 'shareBrowserContextHint' | 'denoise' | 'denoiseHint' | 'maxConcurrency' | 'maxConcurrencyHint' | 'maxConcurrencyPlaceholder' | 'challengeWaitMs' | 'challengeWaitMsHint' | 'challengeWaitMsPlaceholder' | 'overridden' | 'reset' | 'readOnly' | 'expand' | 'collapse' | 'save' | 'saving' | 'discard' | 'unsaved' | 'saveFailed' | 'invalidText';
|
|
9
9
|
/** This plugin's dictionary namespace, merged into the locale key map. */
|
|
10
10
|
declare module '@deepseek-ai/dsh-client-ui-slots' {
|
|
11
11
|
interface LocaleNamespaceMap {
|