dsh-web-fetch-playwright 0.2.4 → 0.2.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/index.js CHANGED
@@ -8,7 +8,7 @@ import { Readability } from "@mozilla/readability";
8
8
  import TurndownService from "turndown";
9
9
  import { gfm } from "@joplin/turndown-plugin-gfm";
10
10
  import { accessSync, constants, readFileSync, realpathSync, statSync } from "node:fs";
11
- import { dirname, join } from "node:path";
11
+ import { delimiter, dirname, join } from "node:path";
12
12
  import { pathToFileURL } from "node:url";
13
13
  //#region src/config.ts
14
14
  /**
@@ -21,13 +21,26 @@ import { pathToFileURL } from "node:url";
21
21
  */
22
22
  /** Default CDP endpoint when the settings section leaves it blank. */
23
23
  const DEFAULT_CDP_ENDPOINT = "127.0.0.1:9222";
24
+ /**
25
+ * Default bounded wait (ms) for a Cloudflare challenge to clear naturally —
26
+ * the user's real browser passes the verification on its own while the fetch
27
+ * holds the same page and context. Sized to fit the 45s per-fetch deadline
28
+ * with the settle/decode tail (and one retry) still inside it.
29
+ */
30
+ const DEFAULT_CHALLENGE_WAIT_MS = 15e3;
31
+ /** Ceiling the schema accepts for `challengeWaitMs`. */
32
+ const MAX_CHALLENGE_WAIT_MS = 6e4;
33
+ /** Default same-page re-navigation attempts after a challenge wait runs out. */
34
+ const DEFAULT_CHALLENGE_RETRIES = 1;
24
35
  const Config = z.object({
25
36
  backend: z.union([z.const("local"), z.const("cdp")]).default("local"),
26
37
  playwrightPath: z.string().default(""),
27
38
  cdpEndpoint: z.string().default(""),
28
39
  shareBrowserContext: z.boolean().default(true),
29
40
  denoise: z.boolean().default(true),
30
- maxConcurrency: z.number().step(1).min(1).max(200)
41
+ maxConcurrency: z.number().step(1).min(1).max(200),
42
+ challengeWaitMs: z.number().step(100).min(0).max(MAX_CHALLENGE_WAIT_MS).default(DEFAULT_CHALLENGE_WAIT_MS),
43
+ challengeRetries: z.number().step(1).min(0).max(3).default(1)
31
44
  });
32
45
  /**
33
46
  * The concurrency limit a fetch actually runs with: an explicit setting
@@ -42,6 +55,28 @@ function effectiveMaxConcurrency(config) {
42
55
  return config.backend === "cdp" ? 50 : 4;
43
56
  }
44
57
  /**
58
+ * The challenge wait budget a fetch actually runs with: an explicit setting
59
+ * wins, else the schema default. `0` means the challenge path is off.
60
+ *
61
+ * @param config - the resolved settings section (or any partial of it).
62
+ * @returns the bounded wait in milliseconds.
63
+ */
64
+ function effectiveChallengeWaitMs(config) {
65
+ if (typeof config.challengeWaitMs === "number") return config.challengeWaitMs;
66
+ return DEFAULT_CHALLENGE_WAIT_MS;
67
+ }
68
+ /**
69
+ * The same-page retry count a challenge fetch actually runs with: an explicit
70
+ * setting wins, else the schema default.
71
+ *
72
+ * @param config - the resolved settings section (or any partial of it).
73
+ * @returns retries after a wait window runs out.
74
+ */
75
+ function effectiveChallengeRetries(config) {
76
+ if (typeof config.challengeRetries === "number") return config.challengeRetries;
77
+ return 1;
78
+ }
79
+ /**
45
80
  * The context mode a fetch actually runs with: profile (a tab in the remote
46
81
  * browser's real profile) requires the CDP backend — a local launch has no
47
82
  * meaningful shared profile — and the checkbox (which defaults on, so an
@@ -73,6 +108,206 @@ function normalizeCdpEndpoint(input) {
73
108
  return candidate;
74
109
  }
75
110
  //#endregion
111
+ //#region src/challenge.ts
112
+ /**
113
+ * Cloudflare challenge detection: the pure predicates and in-page probe the
114
+ * provider's bounded natural-wait uses. Nothing here talks to a browser,
115
+ * a clock, or the network — every function is a mapping over a response's
116
+ * status/headers or an HTML string, so the whole surface is unit-testable.
117
+ *
118
+ * Detection is layered, strongest signal first:
119
+ *
120
+ * 1. Response-level — Cloudflare documents that EVERY challenge page type
121
+ * (JS challenge, managed challenge, Turnstile interstitial) carries
122
+ * `cf-mitigated: challenge`. That header is the primary verdict; a
123
+ * 403/503 from a `server: cloudflare` HTML response is the fallback for
124
+ * origins that strip it.
125
+ * 2. Content-level — the interstitial's localized `<title>` family
126
+ * ("Just a moment...", "请稍候…", ...) plus structural markers
127
+ * (`/cdn-cgi/challenge-platform/` scripts, `#challenge-*` elements,
128
+ * `cf-chl-widget-` Turnstile frames, `window._cf_chl_opt`). Markers are
129
+ * deliberately structural: a blog post that merely MENTIONS Cloudflare
130
+ * must not classify as a challenge.
131
+ *
132
+ * The security stance (issue #2): this module only DETECTS and classifies.
133
+ * Solving, spoofing, CAPTCHA answering, and cookie lifting live outside the
134
+ * plugin entirely.
135
+ *
136
+ * @module dsh-web-fetch-playwright/challenge
137
+ */
138
+ /** How often the bounded wait probes the live page for challenge markers. */
139
+ const CHALLENGE_POLL_INTERVAL_MS = 500;
140
+ /**
141
+ * Tail of the fetch budget held back from the challenge wait so a cleared
142
+ * challenge still has time for its settle (5s networkidle cap) plus the
143
+ * content read and the denoise pipeline.
144
+ */
145
+ const CHALLENGE_FINISH_RESERVE_MS = 8e3;
146
+ /**
147
+ * The interstitial's localized `<title>` cores, anchored full-match with
148
+ * optional trailing ellipsis/punctuation. Cf. ships dozens of locales; this
149
+ * set covers the major ones — the response header remains the primary
150
+ * signal, so the list only needs to catch header-stripping edges.
151
+ */
152
+ const CHALLENGE_TITLE_CORES = [
153
+ "just a moment",
154
+ "einen moment",
155
+ "un instant",
156
+ "un momento",
157
+ "um momento",
158
+ "een ogenblik geduld",
159
+ "минуточку",
160
+ "минутку",
161
+ "пожалуйста, подождите",
162
+ "хвилинку",
163
+ "chwileczkę",
164
+ "chvilku strpení",
165
+ "bir dakika",
166
+ "o clipă",
167
+ "μια στιγμή",
168
+ "少し時間がかかります",
169
+ "잠시만요",
170
+ "잠시 기다려주세요",
171
+ "请稍候",
172
+ "請稍候",
173
+ "เดี๋ยวก่อน",
174
+ "vui lòng chờ",
175
+ "chờ một chút",
176
+ "tunggu sebentar",
177
+ "لحظة من فضلك",
178
+ "الرجاء الانتظار",
179
+ "רק רגע",
180
+ "ett ögonblick",
181
+ "et øjeblik",
182
+ "et øyeblikk",
183
+ "hetkinen",
184
+ "pikainen hetki",
185
+ "egy pillanat",
186
+ "моля, изчакайте"
187
+ ];
188
+ /** Escape regex metacharacters in a literal title core. */
189
+ function escapeRegExp(text) {
190
+ return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
191
+ }
192
+ /**
193
+ * Anchored, case-insensitive match for the localized interstitial title:
194
+ * one of the cores, then only ellipsis/dot/punctuation noise to the end.
195
+ */
196
+ const CHALLENGE_TITLE_RE = new RegExp(`^(?:${CHALLENGE_TITLE_CORES.map(escapeRegExp).join("|")})[\\s.!??…。!]*$`, "i");
197
+ /**
198
+ * Structural markers a challenge document carries in its HTML. Kept in
199
+ * attribute/assignment form (`id="…"`, `window._cf_chl_opt`) rather than bare
200
+ * substrings so an article that merely QUOTES these strings in prose or code
201
+ * samples does not classify — the marker must look like the real element.
202
+ */
203
+ const CHALLENGE_HTML_MARKERS = [
204
+ "/cdn-cgi/challenge-platform/",
205
+ "/cdn-cgi/scripts/jsd/main.js",
206
+ "id=\"cf-chl-widget-",
207
+ "window._cf_chl_opt",
208
+ "id=\"challenge-form\"",
209
+ "id=\"challenge-running\"",
210
+ "id=\"challenge-stage\"",
211
+ "id=\"challenge-error-text\"",
212
+ "name=\"cf-challenge\"",
213
+ "checking your browser before accessing"
214
+ ];
215
+ /** Title cores of the hard-block page ("Attention Required! | Cloudflare"). */
216
+ const BLOCKED_TITLE_RE = /^attention required!/i;
217
+ /**
218
+ * The hard-block page's body phrase, gated by the structural markers below:
219
+ * the phrase alone must not fail a normal article that happens to contain
220
+ * it in prose.
221
+ */
222
+ const BLOCKED_BODY_PHRASE = "you have been blocked";
223
+ /** Structural markers of the hard-block page's classic layout. */
224
+ const BLOCKED_BODY_MARKERS = [
225
+ "cf-headline",
226
+ "cf-error-details",
227
+ "cf-block-details"
228
+ ];
229
+ /** Extract and trim a document's `<title>` text, '' when absent. */
230
+ function titleOf(html) {
231
+ return (/<title[^>]*>([^<]*)<\/title>/i.exec(html)?.[1] ?? "").trim();
232
+ }
233
+ /** Whether a content-type mime is an HTML document (challenge pages are). */
234
+ function isHtmlMime(contentType) {
235
+ const mime = contentType.replace(/;.*$/s, "").trim().toLowerCase();
236
+ return mime === "" || mime === "text/html" || mime === "application/xhtml+xml";
237
+ }
238
+ /**
239
+ * Classify a navigation response from its status and headers.
240
+ *
241
+ * @param status - the response's HTTP status code.
242
+ * @param headers - the response's headers (case as Playwright serves: lowercased).
243
+ * @returns `'challenge'` when the response is a Cloudflare challenge, else `'none'`.
244
+ */
245
+ function classifyChallengeResponse(status, headers) {
246
+ const header = (name) => (headers[name] ?? "").trim().toLowerCase();
247
+ if (header("cf-mitigated") === "challenge") return "challenge";
248
+ if ((status === 403 || status === 503) && header("server").includes("cloudflare") && isHtmlMime(header("content-type"))) return "challenge";
249
+ return "none";
250
+ }
251
+ /**
252
+ * Whether a response is even a candidate for the content-level checks —
253
+ * the suspicion gate. Cloudflare interstitials NEVER ship a plain 200 from
254
+ * a non-Cloudflare origin: they are 403/429/503, or served from a Cloudflare
255
+ * edge (`server: cloudflare`, `cf-ray`). Running the title/marker fallback
256
+ * only on such responses means a normal article — whatever it quotes about
257
+ * Cloudflare, in any language — can never be misread as a challenge. This is
258
+ * the tiering the industry converges on: protocol signals first, content
259
+ * markers only as a gated fallback.
260
+ *
261
+ * @param status - the response's HTTP status code.
262
+ * @param headers - the response's headers (lowercased, as Playwright serves).
263
+ * @returns true when the content-level fallback may run.
264
+ */
265
+ function isChallengeCompatibleResponse(status, headers) {
266
+ if (status === 403 || status === 429 || status === 503) return true;
267
+ const header = (name) => (headers[name] ?? "").trim().toLowerCase();
268
+ return header("server").includes("cloudflare") || header("cf-ray") !== "";
269
+ }
270
+ /**
271
+ * Classify a rendered document from its HTML: the localized interstitial
272
+ * title family and the structural markers above say `challenge`; the
273
+ * hard-block page says `blocked` (waiting cannot clear it — the caller
274
+ * should fail immediately instead of burning its budget).
275
+ *
276
+ * @param html - the document HTML as `page.content()` returns it.
277
+ * @returns the verdict for this HTML.
278
+ */
279
+ function classifyChallengeHtml(html) {
280
+ if (html.length === 0) return "none";
281
+ const title = titleOf(html);
282
+ if (BLOCKED_TITLE_RE.test(title)) return "blocked";
283
+ const lowered = html.toLowerCase();
284
+ if (lowered.includes(BLOCKED_BODY_PHRASE) && BLOCKED_BODY_MARKERS.some((marker) => lowered.includes(marker))) return "blocked";
285
+ if (CHALLENGE_TITLE_RE.test(title)) return "challenge";
286
+ if (CHALLENGE_HTML_MARKERS.some((marker) => lowered.includes(marker))) return "challenge";
287
+ if (lowered.includes("class=\"ray-id\"") && lowered.includes("class=\"footer-inner\"")) return "challenge";
288
+ return "none";
289
+ }
290
+ /**
291
+ * The in-page probe the bounded wait runs against the LIVE document (via
292
+ * `page.evaluate`): true while challenge markers are still present. This is
293
+ * what makes SPA-style clears visible — the document swaps its content
294
+ * without any navigation, so only a DOM read can see the change. The title
295
+ * regex is baked in at build time from the same single source the Node-side
296
+ * check uses, so the two can never drift apart.
297
+ */
298
+ const CHALLENGE_DOM_PROBE = `(() => {
299
+ const re = new RegExp(${JSON.stringify(CHALLENGE_TITLE_RE.source)}, ${JSON.stringify(CHALLENGE_TITLE_RE.flags)})
300
+ if (re.test(document.title.trim())) return true
301
+ if (document.querySelector('#challenge-form, #challenge-running, #challenge-stage, #challenge-error-text')) return true
302
+ if (document.querySelector('[id^="cf-chl-widget-"]')) return true
303
+ for (const script of Array.from(document.scripts)) {
304
+ const src = script.getAttribute('src') || ''
305
+ if (src.includes('/cdn-cgi/challenge-platform/') || src.includes('/cdn-cgi/scripts/jsd/main.js')) return true
306
+ }
307
+ if (document.querySelector('.footer .footer-inner .ray-id')) return true
308
+ return false
309
+ })()`;
310
+ //#endregion
76
311
  //#region src/cdp-pool.ts
77
312
  /** A reusable `connectOverCDP` session handing out per-fetch pages. */
78
313
  var CdpConnectionPool = class {
@@ -458,13 +693,15 @@ const EXECUTABLE_MAGICS = [
458
693
  Buffer.from([77, 90])
459
694
  ];
460
695
  /**
461
- * Find `name` as an executable file on `$PATH`.
696
+ * Find `name` as an executable file on `$PATH`, split on the platform
697
+ * delimiter (`;` on Windows, `:` elsewhere — issue #1: a literal `:` made
698
+ * Windows discovery always miss).
462
699
  * @param name - the executable basename.
463
700
  * @returns the first matching path, or undefined.
464
701
  */
465
702
  function findOnPath(name) {
466
703
  const pathValue = process.env.PATH ?? "";
467
- for (const dir of pathValue.split(":")) {
704
+ for (const dir of pathValue.split(delimiter)) {
468
705
  if (dir === "") continue;
469
706
  const candidate = join(dir, name);
470
707
  try {
@@ -616,10 +853,30 @@ function existsAsFile(path) {
616
853
  * browser can reach. Profile mode additionally acts WITH the remote
617
854
  * browser's logged-in sessions (see the README's risk notes).
618
855
  *
856
+ * Cloudflare challenges (issue #2): when a navigation lands on a challenge
857
+ * interstitial, the fetch waits — on the SAME page and in the SAME browser
858
+ * context, so the browser's natural verification and any clearance cookies
859
+ * it earns apply — for a bounded, configurable window
860
+ * (`challengeWaitMs`, default 15s; 0 restores the legacy first-response
861
+ * behavior). The wait tracks the LAST main-frame navigation response (the
862
+ * real page reloads in after the challenge clears) and watches the live DOM
863
+ * so SPA-style clears are caught too. It never clicks, never injects
864
+ * answers, never touches cookies itself; when the budget runs out the fetch
865
+ * fails with {@link WEB_FETCH_CHALLENGE_CODE} instead of returning the
866
+ * interstitial as content.
867
+ *
619
868
  * @module dsh-web-fetch-playwright/provider
620
869
  */
621
870
  /** Stable id this provider registers under (the bundle patch pins it). */
622
871
  const PLAYWRIGHT_FETCH_PROVIDER_ID = "playwright";
872
+ /**
873
+ * Error code for a Cloudflare challenge the bounded natural wait could not
874
+ * clear. Provider-specific by design — the web seam's `code` is an open
875
+ * string and consumers must tolerate provider-specific codes — so callers
876
+ * can tell "the site challenged us and the browser did not pass" apart from
877
+ * a transport timeout or a provider bug.
878
+ */
879
+ const WEB_FETCH_CHALLENGE_CODE = "WEB_FETCH_CHALLENGE";
623
880
  /** Maximum accepted request URL length (http-provider parity). */
624
881
  const MAX_URL_LENGTH = 2048;
625
882
  /** Cap on the decoded markdown/HTML body this provider returns. */
@@ -891,32 +1148,66 @@ var PlaywrightFetchProvider = class {
891
1148
  throw new WebError(`cannot launch the local browser (${source}); run \`playwright install chromium\` or point the settings path at a playwright/browser executable. ${String(error instanceof Error ? error.message : error)}`, "WEB_PROVIDER_ERROR", { cause: error });
892
1149
  }
893
1150
  }
894
- /** Navigate, settle, and decode one URL inside an open session's tab. */
1151
+ /**
1152
+ * Navigate, settle, and decode one URL inside an open session's tab —
1153
+ * waiting out any Cloudflare interstitial within the bounded challenge
1154
+ * budget before the final document is read (see the class docs).
1155
+ */
895
1156
  async retrieve(session, url, config, deadline) {
896
1157
  const page = session.page;
897
- const response = await page.goto(url.toString(), {
1158
+ const challengeWaitMs = effectiveChallengeWaitMs(config);
1159
+ const tracker = challengeWaitMs > 0 ? trackMainFrameResponses(page) : void 0;
1160
+ let response = await page.goto(url.toString(), {
898
1161
  waitUntil: "domcontentloaded",
899
1162
  timeout: deadline.remainingMs()
900
1163
  });
901
- const kind = classifyContentType(response?.headers()["content-type"]);
902
- if (kind === void 0) throw new WebError(`unsupported content type "${response?.headers()["content-type"] ?? "unknown"}"`, "WEB_UNSUPPORTED_CONTENT_TYPE");
903
- const finalUrl = page.url();
904
- if (kind === "text") {
905
- const text = response !== null ? await response.text() : await page.content();
906
- return capResult(finalUrl, response?.status() ?? 200, {
907
- kind: "text",
908
- content: text
909
- });
1164
+ tracker?.seed(response);
1165
+ let challengeEntryResponse = null;
1166
+ if (challengeWaitMs > 0) {
1167
+ let attemptsLeft = effectiveChallengeRetries(config) + 1;
1168
+ for (;;) {
1169
+ const verdict = await this.verdictAfterLoad(page, tracker?.last() ?? response);
1170
+ if (verdict === "blocked") throw new WebError(`the site hard-blocked this fetch at its Cloudflare edge (waiting cannot clear it): ${page.url()}`, WEB_FETCH_CHALLENGE_CODE);
1171
+ if (verdict !== "challenge") break;
1172
+ challengeEntryResponse = tracker?.last() ?? response;
1173
+ if (await this.waitForChallengeClear(page, deadline, challengeWaitMs)) {
1174
+ let chained = false;
1175
+ try {
1176
+ chained = classifyChallengeHtml(await page.content()) === "challenge";
1177
+ } catch {
1178
+ chained = false;
1179
+ }
1180
+ if (!chained) break;
1181
+ }
1182
+ if (--attemptsLeft <= 0) {
1183
+ const lastStatus = (tracker?.last() ?? response)?.status();
1184
+ throw new WebError(`the site kept serving a Cloudflare challenge (last status ${lastStatus === void 0 ? "unknown" : String(lastStatus)}) for up to ${String(challengeWaitMs)}ms across ${String(effectiveChallengeRetries(config) + 1)} attempt(s); the browser did not clear it naturally — retry later, raise challengeWaitMs, or use a profile whose browser already holds clearance`, WEB_FETCH_CHALLENGE_CODE);
1185
+ }
1186
+ response = await page.goto(url.toString(), {
1187
+ waitUntil: "domcontentloaded",
1188
+ timeout: deadline.remainingMs()
1189
+ });
1190
+ tracker?.seed(response);
1191
+ }
910
1192
  }
1193
+ const finalResponse = (challengeWaitMs > 0 ? tracker?.last() : void 0) ?? response;
1194
+ const kind = classifyContentType(finalResponse?.headers()["content-type"]);
1195
+ if (kind === void 0) throw new WebError(`unsupported content type "${finalResponse?.headers()["content-type"] ?? "unknown"}"`, "WEB_UNSUPPORTED_CONTENT_TYPE");
1196
+ const finalUrl = page.url();
1197
+ const statusCode = finalResponse !== null && !(challengeEntryResponse !== null && finalResponse === challengeEntryResponse) ? finalResponse.status() : 200;
1198
+ if (kind === "text") return capResult(finalUrl, statusCode, {
1199
+ kind: "text",
1200
+ content: finalResponse !== null ? await finalResponse.text() : await page.content()
1201
+ });
911
1202
  await page.waitForLoadState("networkidle", { timeout: Math.min(SETTLE_MS, deadline.remainingMs()) }).catch(() => {});
912
1203
  const html = await page.content();
913
- if (!config.denoise) return capResult(finalUrl, response?.status() ?? 200, {
1204
+ if (!config.denoise) return capResult(finalUrl, statusCode, {
914
1205
  kind: "html",
915
1206
  content: html
916
1207
  });
917
1208
  const bounded = html.length > MAX_PIPELINE_INPUT_CHARS ? html.slice(0, MAX_PIPELINE_INPUT_CHARS) : html;
918
1209
  const { markdown } = htmlToMarkdown(bounded, finalUrl);
919
- const result = capResult(finalUrl, response?.status() ?? 200, {
1210
+ const result = capResult(finalUrl, statusCode, {
920
1211
  kind: "text",
921
1212
  content: markdown
922
1213
  });
@@ -925,7 +1216,111 @@ var PlaywrightFetchProvider = class {
925
1216
  truncated: true
926
1217
  } : result;
927
1218
  }
1219
+ /**
1220
+ * Whether the freshly loaded document is a Cloudflare interstitial: the
1221
+ * response-level signals first (the documented `cf-mitigated` header, then
1222
+ * the 403/503 + cloudflare fallback), then — gated behind
1223
+ * {@link isChallengeCompatibleResponse}, because interstitials never ship
1224
+ * a plain 200 — the localized title family and structural markers, which
1225
+ * also recognize the hard-block page waiting can never clear.
1226
+ */
1227
+ async verdictAfterLoad(page, response) {
1228
+ if (response !== null && classifyChallengeResponse(response.status(), response.headers()) === "challenge") return "challenge";
1229
+ if (response !== null && !isChallengeCompatibleResponse(response.status(), response.headers())) return "none";
1230
+ if (response === null || classifyContentType(response.headers()["content-type"]) === "html") try {
1231
+ return classifyChallengeHtml(await page.content());
1232
+ } catch {
1233
+ return "none";
1234
+ }
1235
+ return "none";
1236
+ }
1237
+ /**
1238
+ * The bounded natural wait: poll the live document for challenge markers
1239
+ * until they are gone (the user's real browser passed the verification and
1240
+ * reloaded into the real page, or swapped it in SPA-style), the per-fetch
1241
+ * wait budget runs out, or the fetch's deadline aborts. No clicks, no
1242
+ * injected answers — the browser either clears it on its own or it does not.
1243
+ *
1244
+ * @returns true when the challenge cleared; false when the budget ran out.
1245
+ */
1246
+ async waitForChallengeClear(page, deadline, challengeWaitMs) {
1247
+ const budget = Math.min(challengeWaitMs, deadline.remainingMs() - CHALLENGE_FINISH_RESERVE_MS);
1248
+ if (budget <= 0) return false;
1249
+ const until = Date.now() + budget;
1250
+ for (;;) {
1251
+ if (deadline.signal.aborted) throw new Error("challenge wait aborted");
1252
+ if (await probeStillOnChallenge(page)) {
1253
+ const remaining = until - Date.now();
1254
+ if (remaining <= 0) return false;
1255
+ await sleep(Math.min(500, remaining));
1256
+ continue;
1257
+ }
1258
+ await page.waitForLoadState("domcontentloaded", { timeout: Math.min(deadline.remainingMs(), 2e3) }).catch(() => {});
1259
+ return true;
1260
+ }
1261
+ }
928
1262
  };
1263
+ /** Resolve after `ms` — the bounded wait's inter-poll nap. */
1264
+ function sleep(ms) {
1265
+ return new Promise((resolve) => {
1266
+ setTimeout(resolve, ms);
1267
+ });
1268
+ }
1269
+ /**
1270
+ * Whether a response event belongs to the page's main frame as a document
1271
+ * navigation — the filter the challenge wait uses to keep the LAST such
1272
+ * response (challenge pages reload the same URL into the real document).
1273
+ * Structural members are optional; when the backend does not expose them the
1274
+ * check degrades to "looks like a document" so fakes stay usable.
1275
+ */
1276
+ function isMainFrameDocument(response, page) {
1277
+ const request = response.request?.();
1278
+ if (request === void 0) return true;
1279
+ if (typeof request.isNavigationRequest === "function" && !request.isNavigationRequest()) return false;
1280
+ const resourceType = typeof request.resourceType === "function" ? request.resourceType() : void 0;
1281
+ if (resourceType !== void 0 && resourceType !== "document") return false;
1282
+ if (typeof request.frame === "function" && typeof page.mainFrame === "function") return request.frame() === page.mainFrame();
1283
+ return true;
1284
+ }
1285
+ /**
1286
+ * Watch the page for main-frame navigation responses so a challenge that
1287
+ * reloads into the real document hands the caller THAT response — status,
1288
+ * headers — instead of the challenge's. Best-effort: a page without the
1289
+ * listener falls back to SPA-style content polling only.
1290
+ */
1291
+ function trackMainFrameResponses(page) {
1292
+ let last = null;
1293
+ try {
1294
+ page.on?.("response", (response) => {
1295
+ if (isMainFrameDocument(response, page)) last = response;
1296
+ });
1297
+ } catch {}
1298
+ return {
1299
+ last: () => last,
1300
+ seed: (response) => {
1301
+ if (response !== null) last = response;
1302
+ }
1303
+ };
1304
+ }
1305
+ /**
1306
+ * Probe the live document for challenge markers. Prefers the in-page probe
1307
+ * (`page.evaluate`) — the only thing that can see SPA-style clears — and
1308
+ * falls back to reading and classifying the serialized HTML when the backend
1309
+ * does not expose evaluate. A probe that throws mid-navigation (execution
1310
+ * context destroyed while the challenge reloads) counts as "still on the
1311
+ * challenge": the next poll sees the fresh document.
1312
+ */
1313
+ async function probeStillOnChallenge(page) {
1314
+ if (typeof page.evaluate === "function") try {
1315
+ const verdict = await page.evaluate(CHALLENGE_DOM_PROBE);
1316
+ if (typeof verdict === "boolean") return verdict;
1317
+ } catch {}
1318
+ try {
1319
+ return classifyChallengeHtml(await page.content()) !== "none";
1320
+ } catch {
1321
+ return true;
1322
+ }
1323
+ }
929
1324
  /** Apply the decoded-body cap and flag the cut. */
930
1325
  function capResult(url, statusCode, body) {
931
1326
  const truncated = body.content.length > MAX_BODY_CHARS;
@@ -1028,4 +1423,4 @@ function apply(ctx, config) {
1028
1423
  ctx.web.registerFetchProvider(provider);
1029
1424
  }
1030
1425
  //#endregion
1031
- export { CdpConnectionPool, Config, DEFAULT_CDP_ENDPOINT, PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider, WEB_FETCH_PLAYWRIGHT_SETTINGS_NAMESPACE, apply, effectiveContextMode, htmlToMarkdown, inject, name, normalizeCdpEndpoint };
1426
+ export { CHALLENGE_DOM_PROBE, CHALLENGE_FINISH_RESERVE_MS, CHALLENGE_POLL_INTERVAL_MS, CHALLENGE_TITLE_RE, CdpConnectionPool, Config, DEFAULT_CDP_ENDPOINT, DEFAULT_CHALLENGE_RETRIES, DEFAULT_CHALLENGE_WAIT_MS, PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider, WEB_FETCH_CHALLENGE_CODE, WEB_FETCH_PLAYWRIGHT_SETTINGS_NAMESPACE, apply, classifyChallengeHtml, classifyChallengeResponse, effectiveChallengeRetries, effectiveChallengeWaitMs, effectiveContextMode, htmlToMarkdown, inject, isChallengeCompatibleResponse, name, normalizeCdpEndpoint };
@@ -0,0 +1,83 @@
1
+ /**
2
+ * Cloudflare challenge detection: the pure predicates and in-page probe the
3
+ * provider's bounded natural-wait uses. Nothing here talks to a browser,
4
+ * a clock, or the network — every function is a mapping over a response's
5
+ * status/headers or an HTML string, so the whole surface is unit-testable.
6
+ *
7
+ * Detection is layered, strongest signal first:
8
+ *
9
+ * 1. Response-level — Cloudflare documents that EVERY challenge page type
10
+ * (JS challenge, managed challenge, Turnstile interstitial) carries
11
+ * `cf-mitigated: challenge`. That header is the primary verdict; a
12
+ * 403/503 from a `server: cloudflare` HTML response is the fallback for
13
+ * origins that strip it.
14
+ * 2. Content-level — the interstitial's localized `<title>` family
15
+ * ("Just a moment...", "请稍候…", ...) plus structural markers
16
+ * (`/cdn-cgi/challenge-platform/` scripts, `#challenge-*` elements,
17
+ * `cf-chl-widget-` Turnstile frames, `window._cf_chl_opt`). Markers are
18
+ * deliberately structural: a blog post that merely MENTIONS Cloudflare
19
+ * must not classify as a challenge.
20
+ *
21
+ * The security stance (issue #2): this module only DETECTS and classifies.
22
+ * Solving, spoofing, CAPTCHA answering, and cookie lifting live outside the
23
+ * plugin entirely.
24
+ *
25
+ * @module dsh-web-fetch-playwright/challenge
26
+ */
27
+ /** How often the bounded wait probes the live page for challenge markers. */
28
+ export declare const CHALLENGE_POLL_INTERVAL_MS = 500;
29
+ /**
30
+ * Tail of the fetch budget held back from the challenge wait so a cleared
31
+ * challenge still has time for its settle (5s networkidle cap) plus the
32
+ * content read and the denoise pipeline.
33
+ */
34
+ export declare const CHALLENGE_FINISH_RESERVE_MS = 8000;
35
+ /**
36
+ * Anchored, case-insensitive match for the localized interstitial title:
37
+ * one of the cores, then only ellipsis/dot/punctuation noise to the end.
38
+ */
39
+ export declare const CHALLENGE_TITLE_RE: RegExp;
40
+ /** The verdicts detection can return. */
41
+ export type ChallengeVerdict = 'none' | 'challenge' | 'blocked';
42
+ /**
43
+ * Classify a navigation response from its status and headers.
44
+ *
45
+ * @param status - the response's HTTP status code.
46
+ * @param headers - the response's headers (case as Playwright serves: lowercased).
47
+ * @returns `'challenge'` when the response is a Cloudflare challenge, else `'none'`.
48
+ */
49
+ export declare function classifyChallengeResponse(status: number, headers: Record<string, string>): 'none' | 'challenge';
50
+ /**
51
+ * Whether a response is even a candidate for the content-level checks —
52
+ * the suspicion gate. Cloudflare interstitials NEVER ship a plain 200 from
53
+ * a non-Cloudflare origin: they are 403/429/503, or served from a Cloudflare
54
+ * edge (`server: cloudflare`, `cf-ray`). Running the title/marker fallback
55
+ * only on such responses means a normal article — whatever it quotes about
56
+ * Cloudflare, in any language — can never be misread as a challenge. This is
57
+ * the tiering the industry converges on: protocol signals first, content
58
+ * markers only as a gated fallback.
59
+ *
60
+ * @param status - the response's HTTP status code.
61
+ * @param headers - the response's headers (lowercased, as Playwright serves).
62
+ * @returns true when the content-level fallback may run.
63
+ */
64
+ export declare function isChallengeCompatibleResponse(status: number, headers: Record<string, string>): boolean;
65
+ /**
66
+ * Classify a rendered document from its HTML: the localized interstitial
67
+ * title family and the structural markers above say `challenge`; the
68
+ * hard-block page says `blocked` (waiting cannot clear it — the caller
69
+ * should fail immediately instead of burning its budget).
70
+ *
71
+ * @param html - the document HTML as `page.content()` returns it.
72
+ * @returns the verdict for this HTML.
73
+ */
74
+ export declare function classifyChallengeHtml(html: string): ChallengeVerdict;
75
+ /**
76
+ * The in-page probe the bounded wait runs against the LIVE document (via
77
+ * `page.evaluate`): true while challenge markers are still present. This is
78
+ * what makes SPA-style clears visible — the document swaps its content
79
+ * without any navigation, so only a DOM read can see the change. The title
80
+ * regex is baked in at build time from the same single source the Node-side
81
+ * check uses, so the two can never drift apart.
82
+ */
83
+ export declare const CHALLENGE_DOM_PROBE: string;
@@ -30,6 +30,11 @@ export interface PlaywrightSettings {
30
30
  denoise?: boolean;
31
31
  /** How many fetches may render at once (1–200); blank = backend default. */
32
32
  maxConcurrency?: number;
33
+ /**
34
+ * Bounded wait (ms) for a Cloudflare challenge to clear naturally in the
35
+ * same tab (0–60000); 0 = off (return the first response as-is).
36
+ */
37
+ challengeWaitMs?: number;
33
38
  }
34
39
  /** What the Playwright card renders. */
35
40
  export interface PlaywrightCardState extends CardShell {
@@ -45,6 +50,8 @@ export interface PlaywrightCardState extends CardShell {
45
50
  denoise: CardFieldState;
46
51
  /** Concurrency input (draft decimal integer). */
47
52
  maxConcurrency: CardFieldState;
53
+ /** Challenge wait input (draft decimal integer of milliseconds). */
54
+ challengeWaitMs: CardFieldState;
48
55
  }
49
56
  /** The registration-side face the card's slot entry injects. */
50
57
  export interface PlaywrightCardFace extends CardActions {
@@ -5,7 +5,7 @@
5
5
  * @module dsh-web-fetch-playwright/client/locales
6
6
  */
7
7
  /** Locale keys this card renders. */
8
- export type PlaywrightCardLocaleKey = 'title' | 'description' | 'backendLabel' | 'backendLocal' | 'backendLocalHint' | 'backendCdp' | 'backendCdpHint' | 'playwrightPath' | 'playwrightPathHint' | 'playwrightPathPlaceholder' | 'cdpEndpoint' | 'cdpEndpointHint' | 'shareBrowserContext' | 'shareBrowserContextHint' | 'denoise' | 'denoiseHint' | 'maxConcurrency' | 'maxConcurrencyHint' | 'maxConcurrencyPlaceholder' | 'overridden' | 'reset' | 'readOnly' | 'expand' | 'collapse' | 'save' | 'saving' | 'discard' | 'unsaved' | 'saveFailed' | 'invalidText';
8
+ export type PlaywrightCardLocaleKey = 'title' | 'description' | 'backendLabel' | 'backendLocal' | 'backendLocalHint' | 'backendCdp' | 'backendCdpHint' | 'playwrightPath' | 'playwrightPathHint' | 'playwrightPathPlaceholder' | 'cdpEndpoint' | 'cdpEndpointHint' | 'shareBrowserContext' | 'shareBrowserContextHint' | 'denoise' | 'denoiseHint' | 'maxConcurrency' | 'maxConcurrencyHint' | 'maxConcurrencyPlaceholder' | 'challengeWaitMs' | 'challengeWaitMsHint' | 'challengeWaitMsPlaceholder' | 'overridden' | 'reset' | 'readOnly' | 'expand' | 'collapse' | 'save' | 'saving' | 'discard' | 'unsaved' | 'saveFailed' | 'invalidText';
9
9
  /** This plugin's dictionary namespace, merged into the locale key map. */
10
10
  declare module '@deepseek-ai/dsh-client-ui-slots' {
11
11
  interface LocaleNamespaceMap {