dsh-web-fetch-playwright 0.2.4 → 0.2.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/index.js CHANGED
@@ -21,13 +21,26 @@ import { pathToFileURL } from "node:url";
21
21
  */
22
22
  /** Default CDP endpoint when the settings section leaves it blank. */
23
23
  const DEFAULT_CDP_ENDPOINT = "127.0.0.1:9222";
24
+ /**
25
+ * Default bounded wait (ms) for a Cloudflare challenge to clear naturally —
26
+ * the user's real browser passes the verification on its own while the fetch
27
+ * holds the same page and context. Sized to fit the 45s per-fetch deadline
28
+ * with the settle/decode tail (and one retry) still inside it.
29
+ */
30
+ const DEFAULT_CHALLENGE_WAIT_MS = 15e3;
31
+ /** Ceiling the schema accepts for `challengeWaitMs`. */
32
+ const MAX_CHALLENGE_WAIT_MS = 6e4;
33
+ /** Default same-page re-navigation attempts after a challenge wait runs out. */
34
+ const DEFAULT_CHALLENGE_RETRIES = 1;
24
35
  const Config = z.object({
25
36
  backend: z.union([z.const("local"), z.const("cdp")]).default("local"),
26
37
  playwrightPath: z.string().default(""),
27
38
  cdpEndpoint: z.string().default(""),
28
39
  shareBrowserContext: z.boolean().default(true),
29
40
  denoise: z.boolean().default(true),
30
- maxConcurrency: z.number().step(1).min(1).max(200)
41
+ maxConcurrency: z.number().step(1).min(1).max(200),
42
+ challengeWaitMs: z.number().step(100).min(0).max(MAX_CHALLENGE_WAIT_MS).default(DEFAULT_CHALLENGE_WAIT_MS),
43
+ challengeRetries: z.number().step(1).min(0).max(3).default(1)
31
44
  });
32
45
  /**
33
46
  * The concurrency limit a fetch actually runs with: an explicit setting
@@ -42,6 +55,28 @@ function effectiveMaxConcurrency(config) {
42
55
  return config.backend === "cdp" ? 50 : 4;
43
56
  }
44
57
  /**
58
+ * The challenge wait budget a fetch actually runs with: an explicit setting
59
+ * wins, else the schema default. `0` means the challenge path is off.
60
+ *
61
+ * @param config - the resolved settings section (or any partial of it).
62
+ * @returns the bounded wait in milliseconds.
63
+ */
64
+ function effectiveChallengeWaitMs(config) {
65
+ if (typeof config.challengeWaitMs === "number") return config.challengeWaitMs;
66
+ return DEFAULT_CHALLENGE_WAIT_MS;
67
+ }
68
+ /**
69
+ * The same-page retry count a challenge fetch actually runs with: an explicit
70
+ * setting wins, else the schema default.
71
+ *
72
+ * @param config - the resolved settings section (or any partial of it).
73
+ * @returns retries after a wait window runs out.
74
+ */
75
+ function effectiveChallengeRetries(config) {
76
+ if (typeof config.challengeRetries === "number") return config.challengeRetries;
77
+ return 1;
78
+ }
79
+ /**
45
80
  * The context mode a fetch actually runs with: profile (a tab in the remote
46
81
  * browser's real profile) requires the CDP backend — a local launch has no
47
82
  * meaningful shared profile — and the checkbox (which defaults on, so an
@@ -73,6 +108,206 @@ function normalizeCdpEndpoint(input) {
73
108
  return candidate;
74
109
  }
75
110
  //#endregion
111
+ //#region src/challenge.ts
112
+ /**
113
+ * Cloudflare challenge detection: the pure predicates and in-page probe the
114
+ * provider's bounded natural-wait uses. Nothing here talks to a browser,
115
+ * a clock, or the network — every function is a mapping over a response's
116
+ * status/headers or an HTML string, so the whole surface is unit-testable.
117
+ *
118
+ * Detection is layered, strongest signal first:
119
+ *
120
+ * 1. Response-level — Cloudflare documents that EVERY challenge page type
121
+ * (JS challenge, managed challenge, Turnstile interstitial) carries
122
+ * `cf-mitigated: challenge`. That header is the primary verdict; a
123
+ * 403/503 from a `server: cloudflare` HTML response is the fallback for
124
+ * origins that strip it.
125
+ * 2. Content-level — the interstitial's localized `<title>` family
126
+ * ("Just a moment...", "请稍候…", ...) plus structural markers
127
+ * (`/cdn-cgi/challenge-platform/` scripts, `#challenge-*` elements,
128
+ * `cf-chl-widget-` Turnstile frames, `window._cf_chl_opt`). Markers are
129
+ * deliberately structural: a blog post that merely MENTIONS Cloudflare
130
+ * must not classify as a challenge.
131
+ *
132
+ * The security stance (issue #2): this module only DETECTS and classifies.
133
+ * Solving, spoofing, CAPTCHA answering, and cookie lifting live outside the
134
+ * plugin entirely.
135
+ *
136
+ * @module dsh-web-fetch-playwright/challenge
137
+ */
138
+ /** How often the bounded wait probes the live page for challenge markers. */
139
+ const CHALLENGE_POLL_INTERVAL_MS = 500;
140
+ /**
141
+ * Tail of the fetch budget held back from the challenge wait so a cleared
142
+ * challenge still has time for its settle (5s networkidle cap) plus the
143
+ * content read and the denoise pipeline.
144
+ */
145
+ const CHALLENGE_FINISH_RESERVE_MS = 8e3;
146
+ /**
147
+ * The interstitial's localized `<title>` cores, anchored full-match with
148
+ * optional trailing ellipsis/punctuation. Cf. ships dozens of locales; this
149
+ * set covers the major ones — the response header remains the primary
150
+ * signal, so the list only needs to catch header-stripping edges.
151
+ */
152
+ const CHALLENGE_TITLE_CORES = [
153
+ "just a moment",
154
+ "einen moment",
155
+ "un instant",
156
+ "un momento",
157
+ "um momento",
158
+ "een ogenblik geduld",
159
+ "минуточку",
160
+ "минутку",
161
+ "пожалуйста, подождите",
162
+ "хвилинку",
163
+ "chwileczkę",
164
+ "chvilku strpení",
165
+ "bir dakika",
166
+ "o clipă",
167
+ "μια στιγμή",
168
+ "少し時間がかかります",
169
+ "잠시만요",
170
+ "잠시 기다려주세요",
171
+ "请稍候",
172
+ "請稍候",
173
+ "เดี๋ยวก่อน",
174
+ "vui lòng chờ",
175
+ "chờ một chút",
176
+ "tunggu sebentar",
177
+ "لحظة من فضلك",
178
+ "الرجاء الانتظار",
179
+ "רק רגע",
180
+ "ett ögonblick",
181
+ "et øjeblik",
182
+ "et øyeblikk",
183
+ "hetkinen",
184
+ "pikainen hetki",
185
+ "egy pillanat",
186
+ "моля, изчакайте"
187
+ ];
188
+ /** Escape regex metacharacters in a literal title core. */
189
+ function escapeRegExp(text) {
190
+ return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
191
+ }
192
+ /**
193
+ * Anchored, case-insensitive match for the localized interstitial title:
194
+ * one of the cores, then only ellipsis/dot/punctuation noise to the end.
195
+ */
196
+ const CHALLENGE_TITLE_RE = new RegExp(`^(?:${CHALLENGE_TITLE_CORES.map(escapeRegExp).join("|")})[\\s.!??…。!]*$`, "i");
197
+ /**
198
+ * Structural markers a challenge document carries in its HTML. Kept in
199
+ * attribute/assignment form (`id="…"`, `window._cf_chl_opt`) rather than bare
200
+ * substrings so an article that merely QUOTES these strings in prose or code
201
+ * samples does not classify — the marker must look like the real element.
202
+ */
203
+ const CHALLENGE_HTML_MARKERS = [
204
+ "/cdn-cgi/challenge-platform/",
205
+ "/cdn-cgi/scripts/jsd/main.js",
206
+ "id=\"cf-chl-widget-",
207
+ "window._cf_chl_opt",
208
+ "id=\"challenge-form\"",
209
+ "id=\"challenge-running\"",
210
+ "id=\"challenge-stage\"",
211
+ "id=\"challenge-error-text\"",
212
+ "name=\"cf-challenge\"",
213
+ "checking your browser before accessing"
214
+ ];
215
+ /** Title cores of the hard-block page ("Attention Required! | Cloudflare"). */
216
+ const BLOCKED_TITLE_RE = /^attention required!/i;
217
+ /**
218
+ * The hard-block page's body phrase, gated by the structural markers below:
219
+ * the phrase alone must not fail a normal article that happens to contain
220
+ * it in prose.
221
+ */
222
+ const BLOCKED_BODY_PHRASE = "you have been blocked";
223
+ /** Structural markers of the hard-block page's classic layout. */
224
+ const BLOCKED_BODY_MARKERS = [
225
+ "cf-headline",
226
+ "cf-error-details",
227
+ "cf-block-details"
228
+ ];
229
+ /** Extract and trim a document's `<title>` text, '' when absent. */
230
+ function titleOf(html) {
231
+ return (/<title[^>]*>([^<]*)<\/title>/i.exec(html)?.[1] ?? "").trim();
232
+ }
233
+ /** Whether a content-type mime is an HTML document (challenge pages are). */
234
+ function isHtmlMime(contentType) {
235
+ const mime = contentType.replace(/;.*$/s, "").trim().toLowerCase();
236
+ return mime === "" || mime === "text/html" || mime === "application/xhtml+xml";
237
+ }
238
+ /**
239
+ * Classify a navigation response from its status and headers.
240
+ *
241
+ * @param status - the response's HTTP status code.
242
+ * @param headers - the response's headers (case as Playwright serves: lowercased).
243
+ * @returns `'challenge'` when the response is a Cloudflare challenge, else `'none'`.
244
+ */
245
+ function classifyChallengeResponse(status, headers) {
246
+ const header = (name) => (headers[name] ?? "").trim().toLowerCase();
247
+ if (header("cf-mitigated") === "challenge") return "challenge";
248
+ if ((status === 403 || status === 503) && header("server").includes("cloudflare") && isHtmlMime(header("content-type"))) return "challenge";
249
+ return "none";
250
+ }
251
+ /**
252
+ * Whether a response is even a candidate for the content-level checks —
253
+ * the suspicion gate. Cloudflare interstitials NEVER ship a plain 200 from
254
+ * a non-Cloudflare origin: they are 403/429/503, or served from a Cloudflare
255
+ * edge (`server: cloudflare`, `cf-ray`). Running the title/marker fallback
256
+ * only on such responses means a normal article — whatever it quotes about
257
+ * Cloudflare, in any language — can never be misread as a challenge. This is
258
+ * the tiering the industry converges on: protocol signals first, content
259
+ * markers only as a gated fallback.
260
+ *
261
+ * @param status - the response's HTTP status code.
262
+ * @param headers - the response's headers (lowercased, as Playwright serves).
263
+ * @returns true when the content-level fallback may run.
264
+ */
265
+ function isChallengeCompatibleResponse(status, headers) {
266
+ if (status === 403 || status === 429 || status === 503) return true;
267
+ const header = (name) => (headers[name] ?? "").trim().toLowerCase();
268
+ return header("server").includes("cloudflare") || header("cf-ray") !== "";
269
+ }
270
+ /**
271
+ * Classify a rendered document from its HTML: the localized interstitial
272
+ * title family and the structural markers above say `challenge`; the
273
+ * hard-block page says `blocked` (waiting cannot clear it — the caller
274
+ * should fail immediately instead of burning its budget).
275
+ *
276
+ * @param html - the document HTML as `page.content()` returns it.
277
+ * @returns the verdict for this HTML.
278
+ */
279
+ function classifyChallengeHtml(html) {
280
+ if (html.length === 0) return "none";
281
+ const title = titleOf(html);
282
+ if (BLOCKED_TITLE_RE.test(title)) return "blocked";
283
+ const lowered = html.toLowerCase();
284
+ if (lowered.includes(BLOCKED_BODY_PHRASE) && BLOCKED_BODY_MARKERS.some((marker) => lowered.includes(marker))) return "blocked";
285
+ if (CHALLENGE_TITLE_RE.test(title)) return "challenge";
286
+ if (CHALLENGE_HTML_MARKERS.some((marker) => lowered.includes(marker))) return "challenge";
287
+ if (lowered.includes("class=\"ray-id\"") && lowered.includes("class=\"footer-inner\"")) return "challenge";
288
+ return "none";
289
+ }
290
+ /**
291
+ * The in-page probe the bounded wait runs against the LIVE document (via
292
+ * `page.evaluate`): true while challenge markers are still present. This is
293
+ * what makes SPA-style clears visible — the document swaps its content
294
+ * without any navigation, so only a DOM read can see the change. The title
295
+ * regex is baked in at build time from the same single source the Node-side
296
+ * check uses, so the two can never drift apart.
297
+ */
298
+ const CHALLENGE_DOM_PROBE = `(() => {
299
+ const re = new RegExp(${JSON.stringify(CHALLENGE_TITLE_RE.source)}, ${JSON.stringify(CHALLENGE_TITLE_RE.flags)})
300
+ if (re.test(document.title.trim())) return true
301
+ if (document.querySelector('#challenge-form, #challenge-running, #challenge-stage, #challenge-error-text')) return true
302
+ if (document.querySelector('[id^="cf-chl-widget-"]')) return true
303
+ for (const script of Array.from(document.scripts)) {
304
+ const src = script.getAttribute('src') || ''
305
+ if (src.includes('/cdn-cgi/challenge-platform/') || src.includes('/cdn-cgi/scripts/jsd/main.js')) return true
306
+ }
307
+ if (document.querySelector('.footer .footer-inner .ray-id')) return true
308
+ return false
309
+ })()`;
310
+ //#endregion
76
311
  //#region src/cdp-pool.ts
77
312
  /** A reusable `connectOverCDP` session handing out per-fetch pages. */
78
313
  var CdpConnectionPool = class {
@@ -616,10 +851,30 @@ function existsAsFile(path) {
616
851
  * browser can reach. Profile mode additionally acts WITH the remote
617
852
  * browser's logged-in sessions (see the README's risk notes).
618
853
  *
854
+ * Cloudflare challenges (issue #2): when a navigation lands on a challenge
855
+ * interstitial, the fetch waits — on the SAME page and in the SAME browser
856
+ * context, so the browser's natural verification and any clearance cookies
857
+ * it earns apply — for a bounded, configurable window
858
+ * (`challengeWaitMs`, default 15s; 0 restores the legacy first-response
859
+ * behavior). The wait tracks the LAST main-frame navigation response (the
860
+ * real page reloads in after the challenge clears) and watches the live DOM
861
+ * so SPA-style clears are caught too. It never clicks, never injects
862
+ * answers, never touches cookies itself; when the budget runs out the fetch
863
+ * fails with {@link WEB_FETCH_CHALLENGE_CODE} instead of returning the
864
+ * interstitial as content.
865
+ *
619
866
  * @module dsh-web-fetch-playwright/provider
620
867
  */
621
868
  /** Stable id this provider registers under (the bundle patch pins it). */
622
869
  const PLAYWRIGHT_FETCH_PROVIDER_ID = "playwright";
870
+ /**
871
+ * Error code for a Cloudflare challenge the bounded natural wait could not
872
+ * clear. Provider-specific by design — the web seam's `code` is an open
873
+ * string and consumers must tolerate provider-specific codes — so callers
874
+ * can tell "the site challenged us and the browser did not pass" apart from
875
+ * a transport timeout or a provider bug.
876
+ */
877
+ const WEB_FETCH_CHALLENGE_CODE = "WEB_FETCH_CHALLENGE";
623
878
  /** Maximum accepted request URL length (http-provider parity). */
624
879
  const MAX_URL_LENGTH = 2048;
625
880
  /** Cap on the decoded markdown/HTML body this provider returns. */
@@ -891,32 +1146,66 @@ var PlaywrightFetchProvider = class {
891
1146
  throw new WebError(`cannot launch the local browser (${source}); run \`playwright install chromium\` or point the settings path at a playwright/browser executable. ${String(error instanceof Error ? error.message : error)}`, "WEB_PROVIDER_ERROR", { cause: error });
892
1147
  }
893
1148
  }
894
- /** Navigate, settle, and decode one URL inside an open session's tab. */
1149
+ /**
1150
+ * Navigate, settle, and decode one URL inside an open session's tab —
1151
+ * waiting out any Cloudflare interstitial within the bounded challenge
1152
+ * budget before the final document is read (see the class docs).
1153
+ */
895
1154
  async retrieve(session, url, config, deadline) {
896
1155
  const page = session.page;
897
- const response = await page.goto(url.toString(), {
1156
+ const challengeWaitMs = effectiveChallengeWaitMs(config);
1157
+ const tracker = challengeWaitMs > 0 ? trackMainFrameResponses(page) : void 0;
1158
+ let response = await page.goto(url.toString(), {
898
1159
  waitUntil: "domcontentloaded",
899
1160
  timeout: deadline.remainingMs()
900
1161
  });
901
- const kind = classifyContentType(response?.headers()["content-type"]);
902
- if (kind === void 0) throw new WebError(`unsupported content type "${response?.headers()["content-type"] ?? "unknown"}"`, "WEB_UNSUPPORTED_CONTENT_TYPE");
903
- const finalUrl = page.url();
904
- if (kind === "text") {
905
- const text = response !== null ? await response.text() : await page.content();
906
- return capResult(finalUrl, response?.status() ?? 200, {
907
- kind: "text",
908
- content: text
909
- });
1162
+ tracker?.seed(response);
1163
+ let challengeEntryResponse = null;
1164
+ if (challengeWaitMs > 0) {
1165
+ let attemptsLeft = effectiveChallengeRetries(config) + 1;
1166
+ for (;;) {
1167
+ const verdict = await this.verdictAfterLoad(page, tracker?.last() ?? response);
1168
+ if (verdict === "blocked") throw new WebError(`the site hard-blocked this fetch at its Cloudflare edge (waiting cannot clear it): ${page.url()}`, WEB_FETCH_CHALLENGE_CODE);
1169
+ if (verdict !== "challenge") break;
1170
+ challengeEntryResponse = tracker?.last() ?? response;
1171
+ if (await this.waitForChallengeClear(page, deadline, challengeWaitMs)) {
1172
+ let chained = false;
1173
+ try {
1174
+ chained = classifyChallengeHtml(await page.content()) === "challenge";
1175
+ } catch {
1176
+ chained = false;
1177
+ }
1178
+ if (!chained) break;
1179
+ }
1180
+ if (--attemptsLeft <= 0) {
1181
+ const lastStatus = (tracker?.last() ?? response)?.status();
1182
+ throw new WebError(`the site kept serving a Cloudflare challenge (last status ${lastStatus === void 0 ? "unknown" : String(lastStatus)}) for up to ${String(challengeWaitMs)}ms across ${String(effectiveChallengeRetries(config) + 1)} attempt(s); the browser did not clear it naturally — retry later, raise challengeWaitMs, or use a profile whose browser already holds clearance`, WEB_FETCH_CHALLENGE_CODE);
1183
+ }
1184
+ response = await page.goto(url.toString(), {
1185
+ waitUntil: "domcontentloaded",
1186
+ timeout: deadline.remainingMs()
1187
+ });
1188
+ tracker?.seed(response);
1189
+ }
910
1190
  }
1191
+ const finalResponse = (challengeWaitMs > 0 ? tracker?.last() : void 0) ?? response;
1192
+ const kind = classifyContentType(finalResponse?.headers()["content-type"]);
1193
+ if (kind === void 0) throw new WebError(`unsupported content type "${finalResponse?.headers()["content-type"] ?? "unknown"}"`, "WEB_UNSUPPORTED_CONTENT_TYPE");
1194
+ const finalUrl = page.url();
1195
+ const statusCode = finalResponse !== null && !(challengeEntryResponse !== null && finalResponse === challengeEntryResponse) ? finalResponse.status() : 200;
1196
+ if (kind === "text") return capResult(finalUrl, statusCode, {
1197
+ kind: "text",
1198
+ content: finalResponse !== null ? await finalResponse.text() : await page.content()
1199
+ });
911
1200
  await page.waitForLoadState("networkidle", { timeout: Math.min(SETTLE_MS, deadline.remainingMs()) }).catch(() => {});
912
1201
  const html = await page.content();
913
- if (!config.denoise) return capResult(finalUrl, response?.status() ?? 200, {
1202
+ if (!config.denoise) return capResult(finalUrl, statusCode, {
914
1203
  kind: "html",
915
1204
  content: html
916
1205
  });
917
1206
  const bounded = html.length > MAX_PIPELINE_INPUT_CHARS ? html.slice(0, MAX_PIPELINE_INPUT_CHARS) : html;
918
1207
  const { markdown } = htmlToMarkdown(bounded, finalUrl);
919
- const result = capResult(finalUrl, response?.status() ?? 200, {
1208
+ const result = capResult(finalUrl, statusCode, {
920
1209
  kind: "text",
921
1210
  content: markdown
922
1211
  });
@@ -925,7 +1214,111 @@ var PlaywrightFetchProvider = class {
925
1214
  truncated: true
926
1215
  } : result;
927
1216
  }
1217
+ /**
1218
+ * Whether the freshly loaded document is a Cloudflare interstitial: the
1219
+ * response-level signals first (the documented `cf-mitigated` header, then
1220
+ * the 403/503 + cloudflare fallback), then — gated behind
1221
+ * {@link isChallengeCompatibleResponse}, because interstitials never ship
1222
+ * a plain 200 — the localized title family and structural markers, which
1223
+ * also recognize the hard-block page waiting can never clear.
1224
+ */
1225
+ async verdictAfterLoad(page, response) {
1226
+ if (response !== null && classifyChallengeResponse(response.status(), response.headers()) === "challenge") return "challenge";
1227
+ if (response !== null && !isChallengeCompatibleResponse(response.status(), response.headers())) return "none";
1228
+ if (response === null || classifyContentType(response.headers()["content-type"]) === "html") try {
1229
+ return classifyChallengeHtml(await page.content());
1230
+ } catch {
1231
+ return "none";
1232
+ }
1233
+ return "none";
1234
+ }
1235
+ /**
1236
+ * The bounded natural wait: poll the live document for challenge markers
1237
+ * until they are gone (the user's real browser passed the verification and
1238
+ * reloaded into the real page, or swapped it in SPA-style), the per-fetch
1239
+ * wait budget runs out, or the fetch's deadline aborts. No clicks, no
1240
+ * injected answers — the browser either clears it on its own or it does not.
1241
+ *
1242
+ * @returns true when the challenge cleared; false when the budget ran out.
1243
+ */
1244
+ async waitForChallengeClear(page, deadline, challengeWaitMs) {
1245
+ const budget = Math.min(challengeWaitMs, deadline.remainingMs() - CHALLENGE_FINISH_RESERVE_MS);
1246
+ if (budget <= 0) return false;
1247
+ const until = Date.now() + budget;
1248
+ for (;;) {
1249
+ if (deadline.signal.aborted) throw new Error("challenge wait aborted");
1250
+ if (await probeStillOnChallenge(page)) {
1251
+ const remaining = until - Date.now();
1252
+ if (remaining <= 0) return false;
1253
+ await sleep(Math.min(500, remaining));
1254
+ continue;
1255
+ }
1256
+ await page.waitForLoadState("domcontentloaded", { timeout: Math.min(deadline.remainingMs(), 2e3) }).catch(() => {});
1257
+ return true;
1258
+ }
1259
+ }
928
1260
  };
1261
+ /** Resolve after `ms` — the bounded wait's inter-poll nap. */
1262
+ function sleep(ms) {
1263
+ return new Promise((resolve) => {
1264
+ setTimeout(resolve, ms);
1265
+ });
1266
+ }
1267
+ /**
1268
+ * Whether a response event belongs to the page's main frame as a document
1269
+ * navigation — the filter the challenge wait uses to keep the LAST such
1270
+ * response (challenge pages reload the same URL into the real document).
1271
+ * Structural members are optional; when the backend does not expose them the
1272
+ * check degrades to "looks like a document" so fakes stay usable.
1273
+ */
1274
+ function isMainFrameDocument(response, page) {
1275
+ const request = response.request?.();
1276
+ if (request === void 0) return true;
1277
+ if (typeof request.isNavigationRequest === "function" && !request.isNavigationRequest()) return false;
1278
+ const resourceType = typeof request.resourceType === "function" ? request.resourceType() : void 0;
1279
+ if (resourceType !== void 0 && resourceType !== "document") return false;
1280
+ if (typeof request.frame === "function" && typeof page.mainFrame === "function") return request.frame() === page.mainFrame();
1281
+ return true;
1282
+ }
1283
+ /**
1284
+ * Watch the page for main-frame navigation responses so a challenge that
1285
+ * reloads into the real document hands the caller THAT response — status,
1286
+ * headers — instead of the challenge's. Best-effort: a page without the
1287
+ * listener falls back to SPA-style content polling only.
1288
+ */
1289
+ function trackMainFrameResponses(page) {
1290
+ let last = null;
1291
+ try {
1292
+ page.on?.("response", (response) => {
1293
+ if (isMainFrameDocument(response, page)) last = response;
1294
+ });
1295
+ } catch {}
1296
+ return {
1297
+ last: () => last,
1298
+ seed: (response) => {
1299
+ if (response !== null) last = response;
1300
+ }
1301
+ };
1302
+ }
1303
+ /**
1304
+ * Probe the live document for challenge markers. Prefers the in-page probe
1305
+ * (`page.evaluate`) — the only thing that can see SPA-style clears — and
1306
+ * falls back to reading and classifying the serialized HTML when the backend
1307
+ * does not expose evaluate. A probe that throws mid-navigation (execution
1308
+ * context destroyed while the challenge reloads) counts as "still on the
1309
+ * challenge": the next poll sees the fresh document.
1310
+ */
1311
+ async function probeStillOnChallenge(page) {
1312
+ if (typeof page.evaluate === "function") try {
1313
+ const verdict = await page.evaluate(CHALLENGE_DOM_PROBE);
1314
+ if (typeof verdict === "boolean") return verdict;
1315
+ } catch {}
1316
+ try {
1317
+ return classifyChallengeHtml(await page.content()) !== "none";
1318
+ } catch {
1319
+ return true;
1320
+ }
1321
+ }
929
1322
  /** Apply the decoded-body cap and flag the cut. */
930
1323
  function capResult(url, statusCode, body) {
931
1324
  const truncated = body.content.length > MAX_BODY_CHARS;
@@ -1028,4 +1421,4 @@ function apply(ctx, config) {
1028
1421
  ctx.web.registerFetchProvider(provider);
1029
1422
  }
1030
1423
  //#endregion
1031
- export { CdpConnectionPool, Config, DEFAULT_CDP_ENDPOINT, PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider, WEB_FETCH_PLAYWRIGHT_SETTINGS_NAMESPACE, apply, effectiveContextMode, htmlToMarkdown, inject, name, normalizeCdpEndpoint };
1424
+ export { CHALLENGE_DOM_PROBE, CHALLENGE_FINISH_RESERVE_MS, CHALLENGE_POLL_INTERVAL_MS, CHALLENGE_TITLE_RE, CdpConnectionPool, Config, DEFAULT_CDP_ENDPOINT, DEFAULT_CHALLENGE_RETRIES, DEFAULT_CHALLENGE_WAIT_MS, PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider, WEB_FETCH_CHALLENGE_CODE, WEB_FETCH_PLAYWRIGHT_SETTINGS_NAMESPACE, apply, classifyChallengeHtml, classifyChallengeResponse, effectiveChallengeRetries, effectiveChallengeWaitMs, effectiveContextMode, htmlToMarkdown, inject, isChallengeCompatibleResponse, name, normalizeCdpEndpoint };
@@ -0,0 +1,83 @@
1
+ /**
2
+ * Cloudflare challenge detection: the pure predicates and in-page probe the
3
+ * provider's bounded natural-wait uses. Nothing here talks to a browser,
4
+ * a clock, or the network — every function is a mapping over a response's
5
+ * status/headers or an HTML string, so the whole surface is unit-testable.
6
+ *
7
+ * Detection is layered, strongest signal first:
8
+ *
9
+ * 1. Response-level — Cloudflare documents that EVERY challenge page type
10
+ * (JS challenge, managed challenge, Turnstile interstitial) carries
11
+ * `cf-mitigated: challenge`. That header is the primary verdict; a
12
+ * 403/503 from a `server: cloudflare` HTML response is the fallback for
13
+ * origins that strip it.
14
+ * 2. Content-level — the interstitial's localized `<title>` family
15
+ * ("Just a moment...", "请稍候…", ...) plus structural markers
16
+ * (`/cdn-cgi/challenge-platform/` scripts, `#challenge-*` elements,
17
+ * `cf-chl-widget-` Turnstile frames, `window._cf_chl_opt`). Markers are
18
+ * deliberately structural: a blog post that merely MENTIONS Cloudflare
19
+ * must not classify as a challenge.
20
+ *
21
+ * The security stance (issue #2): this module only DETECTS and classifies.
22
+ * Solving, spoofing, CAPTCHA answering, and cookie lifting live outside the
23
+ * plugin entirely.
24
+ *
25
+ * @module dsh-web-fetch-playwright/challenge
26
+ */
27
+ /** How often the bounded wait probes the live page for challenge markers. */
28
+ export declare const CHALLENGE_POLL_INTERVAL_MS = 500;
29
+ /**
30
+ * Tail of the fetch budget held back from the challenge wait so a cleared
31
+ * challenge still has time for its settle (5s networkidle cap) plus the
32
+ * content read and the denoise pipeline.
33
+ */
34
+ export declare const CHALLENGE_FINISH_RESERVE_MS = 8000;
35
+ /**
36
+ * Anchored, case-insensitive match for the localized interstitial title:
37
+ * one of the cores, then only ellipsis/dot/punctuation noise to the end.
38
+ */
39
+ export declare const CHALLENGE_TITLE_RE: RegExp;
40
+ /** The verdicts detection can return. */
41
+ export type ChallengeVerdict = 'none' | 'challenge' | 'blocked';
42
+ /**
43
+ * Classify a navigation response from its status and headers.
44
+ *
45
+ * @param status - the response's HTTP status code.
46
+ * @param headers - the response's headers (case as Playwright serves: lowercased).
47
+ * @returns `'challenge'` when the response is a Cloudflare challenge, else `'none'`.
48
+ */
49
+ export declare function classifyChallengeResponse(status: number, headers: Record<string, string>): 'none' | 'challenge';
50
+ /**
51
+ * Whether a response is even a candidate for the content-level checks —
52
+ * the suspicion gate. Cloudflare interstitials NEVER ship a plain 200 from
53
+ * a non-Cloudflare origin: they are 403/429/503, or served from a Cloudflare
54
+ * edge (`server: cloudflare`, `cf-ray`). Running the title/marker fallback
55
+ * only on such responses means a normal article — whatever it quotes about
56
+ * Cloudflare, in any language — can never be misread as a challenge. This is
57
+ * the tiering the industry converges on: protocol signals first, content
58
+ * markers only as a gated fallback.
59
+ *
60
+ * @param status - the response's HTTP status code.
61
+ * @param headers - the response's headers (lowercased, as Playwright serves).
62
+ * @returns true when the content-level fallback may run.
63
+ */
64
+ export declare function isChallengeCompatibleResponse(status: number, headers: Record<string, string>): boolean;
65
+ /**
66
+ * Classify a rendered document from its HTML: the localized interstitial
67
+ * title family and the structural markers above say `challenge`; the
68
+ * hard-block page says `blocked` (waiting cannot clear it — the caller
69
+ * should fail immediately instead of burning its budget).
70
+ *
71
+ * @param html - the document HTML as `page.content()` returns it.
72
+ * @returns the verdict for this HTML.
73
+ */
74
+ export declare function classifyChallengeHtml(html: string): ChallengeVerdict;
75
+ /**
76
+ * The in-page probe the bounded wait runs against the LIVE document (via
77
+ * `page.evaluate`): true while challenge markers are still present. This is
78
+ * what makes SPA-style clears visible — the document swaps its content
79
+ * without any navigation, so only a DOM read can see the change. The title
80
+ * regex is baked in at build time from the same single source the Node-side
81
+ * check uses, so the two can never drift apart.
82
+ */
83
+ export declare const CHALLENGE_DOM_PROBE: string;
@@ -30,6 +30,11 @@ export interface PlaywrightSettings {
30
30
  denoise?: boolean;
31
31
  /** How many fetches may render at once (1–200); blank = backend default. */
32
32
  maxConcurrency?: number;
33
+ /**
34
+ * Bounded wait (ms) for a Cloudflare challenge to clear naturally in the
35
+ * same tab (0–60000); 0 = off (return the first response as-is).
36
+ */
37
+ challengeWaitMs?: number;
33
38
  }
34
39
  /** What the Playwright card renders. */
35
40
  export interface PlaywrightCardState extends CardShell {
@@ -45,6 +50,8 @@ export interface PlaywrightCardState extends CardShell {
45
50
  denoise: CardFieldState;
46
51
  /** Concurrency input (draft decimal integer). */
47
52
  maxConcurrency: CardFieldState;
53
+ /** Challenge wait input (draft decimal integer of milliseconds). */
54
+ challengeWaitMs: CardFieldState;
48
55
  }
49
56
  /** The registration-side face the card's slot entry injects. */
50
57
  export interface PlaywrightCardFace extends CardActions {
@@ -5,7 +5,7 @@
5
5
  * @module dsh-web-fetch-playwright/client/locales
6
6
  */
7
7
  /** Locale keys this card renders. */
8
- export type PlaywrightCardLocaleKey = 'title' | 'description' | 'backendLabel' | 'backendLocal' | 'backendLocalHint' | 'backendCdp' | 'backendCdpHint' | 'playwrightPath' | 'playwrightPathHint' | 'playwrightPathPlaceholder' | 'cdpEndpoint' | 'cdpEndpointHint' | 'shareBrowserContext' | 'shareBrowserContextHint' | 'denoise' | 'denoiseHint' | 'maxConcurrency' | 'maxConcurrencyHint' | 'maxConcurrencyPlaceholder' | 'overridden' | 'reset' | 'readOnly' | 'expand' | 'collapse' | 'save' | 'saving' | 'discard' | 'unsaved' | 'saveFailed' | 'invalidText';
8
+ export type PlaywrightCardLocaleKey = 'title' | 'description' | 'backendLabel' | 'backendLocal' | 'backendLocalHint' | 'backendCdp' | 'backendCdpHint' | 'playwrightPath' | 'playwrightPathHint' | 'playwrightPathPlaceholder' | 'cdpEndpoint' | 'cdpEndpointHint' | 'shareBrowserContext' | 'shareBrowserContextHint' | 'denoise' | 'denoiseHint' | 'maxConcurrency' | 'maxConcurrencyHint' | 'maxConcurrencyPlaceholder' | 'challengeWaitMs' | 'challengeWaitMsHint' | 'challengeWaitMsPlaceholder' | 'overridden' | 'reset' | 'readOnly' | 'expand' | 'collapse' | 'save' | 'saving' | 'discard' | 'unsaved' | 'saveFailed' | 'invalidText';
9
9
  /** This plugin's dictionary namespace, merged into the locale key map. */
10
10
  declare module '@deepseek-ai/dsh-client-ui-slots' {
11
11
  interface LocaleNamespaceMap {
@@ -24,6 +24,19 @@ export declare const DEFAULT_MAX_CONCURRENCY_LOCAL = 4;
24
24
  export declare const DEFAULT_MAX_CONCURRENCY_CDP = 50;
25
25
  /** Ceiling the schema accepts for `maxConcurrency` (local slots are browsers). */
26
26
  export declare const MAX_CONCURRENCY_CEILING = 200;
27
+ /**
28
+ * Default bounded wait (ms) for a Cloudflare challenge to clear naturally —
29
+ * the user's real browser passes the verification on its own while the fetch
30
+ * holds the same page and context. Sized to fit the 45s per-fetch deadline
31
+ * with the settle/decode tail (and one retry) still inside it.
32
+ */
33
+ export declare const DEFAULT_CHALLENGE_WAIT_MS = 15000;
34
+ /** Ceiling the schema accepts for `challengeWaitMs`. */
35
+ export declare const MAX_CHALLENGE_WAIT_MS = 60000;
36
+ /** Default same-page re-navigation attempts after a challenge wait runs out. */
37
+ export declare const DEFAULT_CHALLENGE_RETRIES = 1;
38
+ /** Ceiling the schema accepts for `challengeRetries`. */
39
+ export declare const MAX_CHALLENGE_RETRIES = 3;
27
40
  /** Which browser backend serves a fetch. */
28
41
  export type PlaywrightBackend = 'local' | 'cdp';
29
42
  /**
@@ -53,6 +66,19 @@ export interface Config {
53
66
  shareBrowserContext?: boolean;
54
67
  /** Whether the Readability + DOMPurify denoise pipeline runs before markdown. */
55
68
  denoise?: boolean;
69
+ /**
70
+ * Bounded wait (ms) for a Cloudflare challenge to clear naturally inside
71
+ * the same page/context. `0` (or any config leaving this at 0) restores the
72
+ * legacy behavior: the first response is final, no waiting — the A/B
73
+ * baseline. Schema default: {@link DEFAULT_CHALLENGE_WAIT_MS}.
74
+ */
75
+ challengeWaitMs?: number;
76
+ /**
77
+ * Same-page re-navigation attempts after a challenge wait window runs out
78
+ * (any partial clearance cookies stay in the context for the retry).
79
+ * Schema default: {@link DEFAULT_CHALLENGE_RETRIES}.
80
+ */
81
+ challengeRetries?: number;
56
82
  /**
57
83
  * How many fetches may render at once. Blank = backend default (4 for
58
84
  * local — each slot launches a browser; 50 for CDP — each slot is a tab in
@@ -78,6 +104,22 @@ export type ResolvedConfig = Omit<Required<Config>, 'maxConcurrency'> & {
78
104
  * @returns the effective limit for the semaphore.
79
105
  */
80
106
  export declare function effectiveMaxConcurrency(config: Pick<Config, 'backend' | 'maxConcurrency'>): number;
107
+ /**
108
+ * The challenge wait budget a fetch actually runs with: an explicit setting
109
+ * wins, else the schema default. `0` means the challenge path is off.
110
+ *
111
+ * @param config - the resolved settings section (or any partial of it).
112
+ * @returns the bounded wait in milliseconds.
113
+ */
114
+ export declare function effectiveChallengeWaitMs(config: Pick<Config, 'challengeWaitMs'>): number;
115
+ /**
116
+ * The same-page retry count a challenge fetch actually runs with: an explicit
117
+ * setting wins, else the schema default.
118
+ *
119
+ * @param config - the resolved settings section (or any partial of it).
120
+ * @returns retries after a wait window runs out.
121
+ */
122
+ export declare function effectiveChallengeRetries(config: Pick<Config, 'challengeRetries'>): number;
81
123
  /**
82
124
  * The context mode a fetch actually runs with: profile (a tab in the remote
83
125
  * browser's real profile) requires the CDP backend — a local launch has no