dsh-web-fetch-playwright 0.2.9 → 0.2.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/README.zh-CN.md +2 -0
- package/lib/client.js +19 -19
- package/lib/index.js +123 -16
- package/lib/types/provider.d.ts +21 -1
- package/package.json +1 -1
- package/src/provider.ts +139 -10
package/README.md
CHANGED
|
@@ -34,6 +34,8 @@ web_fetch (tool-web)
|
|
|
34
34
|
└─ Markdown (or raw HTML when denoise is off)
|
|
35
35
|
```
|
|
36
36
|
|
|
37
|
+
**Non-2xx statuses are results, not errors** — the web seam's contract (`@deepseek-ai/dsh-web`): a fetched 404/418/500 comes back with its status code, exactly what `dsh-web-fetch-http` does. Chromium complicates this for *status-only* responses — an error status with an empty body (httpbin's `/status/404`) fails the whole navigation with `net::ERR_HTTP_RESPONSE_CODE_FAILURE`, and a 204/205 is aborted (`net::ERR_ABORTED`) because nothing is renderable. The provider therefore tracks the last main-frame response on every fetch (that response arrives before Chromium fails the navigation) and recovers such failures into the status-bearing result, so `web_fetch` reports `Fetched … (HTTP 404)` instead of a `WEB_PROVIDER_ERROR` throw. A recovered response that is itself a Cloudflare challenge edge still fails with `WEB_FETCH_CHALLENGE`; transport errors and cancellations stay errors.
|
|
38
|
+
|
|
37
39
|
## Requirements
|
|
38
40
|
|
|
39
41
|
- DSH web profile (`dsh web`), Node.js ≥ 20.
|
package/README.zh-CN.md
CHANGED
|
@@ -34,6 +34,8 @@ web_fetch (tool-web)
|
|
|
34
34
|
└─ Markdown(关闭降噪时返回原始 HTML)
|
|
35
35
|
```
|
|
36
36
|
|
|
37
|
+
**非 2xx 状态码是结果而非错误** —— 这是 web seam 的契约(`@deepseek-ai/dsh-web`):抓取到的 404/418/500 会连同状态码一起正常返回,与 `dsh-web-fetch-http` 的行为一致。但对*仅有状态*的响应 Chromium 会把事情搞砸 —— 空 body 的错误状态(httpbin 的 `/status/404`)会让整个导航以 `net::ERR_HTTP_RESPONSE_CODE_FAILURE` 失败,204/205 则因无可渲染内容被中止(`net::ERR_ABORTED`)。因此 provider 在每次抓取中都跟踪最后一个主框架响应(该响应先于 Chromium 判定导航失败到达),并把这类失败恢复成带状态码的正常结果,`web_fetch` 会报告 `Fetched … (HTTP 404)` 而不是抛出 `WEB_PROVIDER_ERROR`。若恢复出的响应本身来自 Cloudflare 挑战边缘,仍以 `WEB_FETCH_CHALLENGE` 失败;传输错误与取消也依旧按错误处理。
|
|
38
|
+
|
|
37
39
|
## 环境要求
|
|
38
40
|
|
|
39
41
|
- DSH web profile(`dsh web`),Node.js ≥ 20。
|
package/lib/client.js
CHANGED
|
@@ -412,12 +412,12 @@ window.__ModuleLoader__.load({
|
|
|
412
412
|
document.head.appendChild(tag);
|
|
413
413
|
}
|
|
414
414
|
var PluginCard_module_css_default = {
|
|
415
|
-
"
|
|
416
|
-
"save": "UM8-Rq_save",
|
|
417
|
-
"footer": "UM8-Rq_footer",
|
|
418
|
-
"failed": "UM8-Rq_failed",
|
|
415
|
+
"readOnly": "UM8-Rq_readOnly",
|
|
419
416
|
"form": "UM8-Rq_form",
|
|
420
|
-
"
|
|
417
|
+
"failed": "UM8-Rq_failed",
|
|
418
|
+
"footer": "UM8-Rq_footer",
|
|
419
|
+
"save": "UM8-Rq_save",
|
|
420
|
+
"discard": "UM8-Rq_discard"
|
|
421
421
|
};
|
|
422
422
|
//#endregion
|
|
423
423
|
//#region src/client/PluginCard.tsx
|
|
@@ -477,26 +477,26 @@ window.__ModuleLoader__.load({
|
|
|
477
477
|
document.head.appendChild(tag);
|
|
478
478
|
}
|
|
479
479
|
var fields_module_css_default = {
|
|
480
|
-
"
|
|
481
|
-
"radioGroup": "CFSaVG_radioGroup",
|
|
482
|
-
"fieldEmbedded": "CFSaVG_fieldEmbedded",
|
|
483
|
-
"radioContent": "CFSaVG_radioContent",
|
|
484
|
-
"radioOption": "CFSaVG_radioOption",
|
|
480
|
+
"input": "CFSaVG_input",
|
|
485
481
|
"inputInvalid": "CFSaVG_inputInvalid",
|
|
482
|
+
"hintIndented": "CFSaVG_hintIndented",
|
|
486
483
|
"radioText": "CFSaVG_radioText",
|
|
487
|
-
"
|
|
484
|
+
"radioPick": "CFSaVG_radioPick",
|
|
488
485
|
"radioLabel": "CFSaVG_radioLabel",
|
|
489
|
-
"field": "CFSaVG_field",
|
|
490
|
-
"input": "CFSaVG_input",
|
|
491
|
-
"badges": "CFSaVG_badges",
|
|
492
486
|
"invalid": "CFSaVG_invalid",
|
|
493
|
-
"
|
|
487
|
+
"radioHint": "CFSaVG_radioHint",
|
|
488
|
+
"radioOption": "CFSaVG_radioOption",
|
|
489
|
+
"badges": "CFSaVG_badges",
|
|
494
490
|
"label": "CFSaVG_label",
|
|
491
|
+
"hint": "CFSaVG_hint",
|
|
492
|
+
"badge": "CFSaVG_badge",
|
|
493
|
+
"field": "CFSaVG_field",
|
|
495
494
|
"reset": "CFSaVG_reset",
|
|
496
|
-
"
|
|
497
|
-
"
|
|
498
|
-
"
|
|
499
|
-
"
|
|
495
|
+
"fieldEmbedded": "CFSaVG_fieldEmbedded",
|
|
496
|
+
"radioGroup": "CFSaVG_radioGroup",
|
|
497
|
+
"checkboxRow": "CFSaVG_checkboxRow",
|
|
498
|
+
"head": "CFSaVG_head",
|
|
499
|
+
"radioContent": "CFSaVG_radioContent"
|
|
500
500
|
};
|
|
501
501
|
//#endregion
|
|
502
502
|
//#region src/client/fields.tsx
|
package/lib/index.js
CHANGED
|
@@ -993,7 +993,16 @@ function existsAsFile(path) {
|
|
|
993
993
|
* The Playwright `WebFetchProvider`: renders one URL in a real browser and
|
|
994
994
|
* returns it as markdown (denoised) or HTML. Mirrors `dsh-web-fetch-http`'s
|
|
995
995
|
* error taxonomy (URL hygiene, abort/timeout translation, content-type
|
|
996
|
-
* classification) so the tool layer sees the same codes from either backend
|
|
996
|
+
* classification) so the tool layer sees the same codes from either backend —
|
|
997
|
+
* including the seam contract that a fetched non-2xx status is a RESULT, not
|
|
998
|
+
* an error. Chromium can make that awkward: it refuses to render status-only
|
|
999
|
+
* responses (error statuses with an empty body — httpbin's `/status/N` —
|
|
1000
|
+
* plus 204/205), failing the whole navigation with
|
|
1001
|
+
* `net::ERR_HTTP_RESPONSE_CODE_FAILURE` / `net::ERR_ABORTED` even though the
|
|
1002
|
+
* response itself arrived. The last main-frame response is therefore tracked
|
|
1003
|
+
* on every fetch, and such a failed `goto` is recovered into the
|
|
1004
|
+
* status-bearing result the contract demands instead of a
|
|
1005
|
+
* `WEB_PROVIDER_ERROR` (see {@link recoverUnrenderedStatus}).
|
|
997
1006
|
*
|
|
998
1007
|
* Lifecycle: the local backend launches a browser per fetch and closes it —
|
|
999
1008
|
* nothing outlives the call. The CDP backend keeps ONE shared connection to
|
|
@@ -1316,20 +1325,19 @@ var PlaywrightFetchProvider = class {
|
|
|
1316
1325
|
async retrieve(session, url, config, deadline) {
|
|
1317
1326
|
const page = session.page;
|
|
1318
1327
|
const challengeWaitMs = effectiveChallengeWaitMs(config);
|
|
1319
|
-
const tracker =
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
tracker?.seed(response);
|
|
1328
|
+
const tracker = trackMainFrameResponses(page);
|
|
1329
|
+
const first = await this.navigate(page, url, tracker, deadline, config.denoise);
|
|
1330
|
+
if (first.recovered !== void 0) return first.recovered;
|
|
1331
|
+
let response = first.response;
|
|
1332
|
+
tracker.seed(response);
|
|
1325
1333
|
let challengeEntryResponse = null;
|
|
1326
1334
|
if (challengeWaitMs > 0) {
|
|
1327
1335
|
let attemptsLeft = effectiveChallengeRetries(config) + 1;
|
|
1328
1336
|
for (;;) {
|
|
1329
|
-
const verdict = await this.verdictAfterLoad(page, tracker
|
|
1337
|
+
const verdict = await this.verdictAfterLoad(page, tracker.last() ?? response);
|
|
1330
1338
|
if (verdict === "blocked") throw new WebError(`the site hard-blocked this fetch at its Cloudflare edge (waiting cannot clear it): ${page.url()}`, WEB_FETCH_CHALLENGE_CODE);
|
|
1331
1339
|
if (verdict !== "challenge") break;
|
|
1332
|
-
challengeEntryResponse = tracker
|
|
1340
|
+
challengeEntryResponse = tracker.last() ?? response;
|
|
1333
1341
|
if (await this.waitForChallengeClear(page, deadline, challengeWaitMs)) {
|
|
1334
1342
|
let chained = false;
|
|
1335
1343
|
try {
|
|
@@ -1340,17 +1348,16 @@ var PlaywrightFetchProvider = class {
|
|
|
1340
1348
|
if (!chained) break;
|
|
1341
1349
|
}
|
|
1342
1350
|
if (--attemptsLeft <= 0) {
|
|
1343
|
-
const lastStatus = (tracker
|
|
1351
|
+
const lastStatus = (tracker.last() ?? response)?.status();
|
|
1344
1352
|
throw new WebError(`the site kept serving a Cloudflare challenge (last status ${lastStatus === void 0 ? "unknown" : String(lastStatus)}) for up to ${String(challengeWaitMs)}ms across ${String(effectiveChallengeRetries(config) + 1)} attempt(s); the browser did not clear it naturally — retry later, raise challengeWaitMs, or use a profile whose browser already holds clearance`, WEB_FETCH_CHALLENGE_CODE);
|
|
1345
1353
|
}
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
tracker?.seed(response);
|
|
1354
|
+
const retry = await this.navigate(page, url, tracker, deadline, config.denoise);
|
|
1355
|
+
if (retry.recovered !== void 0) return retry.recovered;
|
|
1356
|
+
response = retry.response;
|
|
1357
|
+
tracker.seed(response);
|
|
1351
1358
|
}
|
|
1352
1359
|
}
|
|
1353
|
-
const finalResponse = (challengeWaitMs > 0 ? tracker
|
|
1360
|
+
const finalResponse = (challengeWaitMs > 0 ? tracker.last() : void 0) ?? response;
|
|
1354
1361
|
const kind = classifyContentType(finalResponse?.headers()["content-type"]);
|
|
1355
1362
|
if (kind === void 0) throw new WebError(`unsupported content type "${finalResponse?.headers()["content-type"] ?? "unknown"}"`, "WEB_UNSUPPORTED_CONTENT_TYPE");
|
|
1356
1363
|
const finalUrl = page.url();
|
|
@@ -1377,6 +1384,33 @@ var PlaywrightFetchProvider = class {
|
|
|
1377
1384
|
} : result;
|
|
1378
1385
|
}
|
|
1379
1386
|
/**
|
|
1387
|
+
* One tracked `goto`: the navigation response on success, or — when
|
|
1388
|
+
* Chromium failed the navigation purely because of the HTTP status it
|
|
1389
|
+
* received (an error status with an empty body, or a status-only 204/205)
|
|
1390
|
+
* — the recovered status-bearing result from {@link recoverUnrenderedStatus}.
|
|
1391
|
+
*
|
|
1392
|
+
* @returns the response `goto` resolved with, plus `recovered` when the
|
|
1393
|
+
* navigation failure was recovered into a finished result the caller must
|
|
1394
|
+
* return verbatim (the throw never escapes in that case).
|
|
1395
|
+
*/
|
|
1396
|
+
async navigate(page, url, tracker, deadline, denoise) {
|
|
1397
|
+
try {
|
|
1398
|
+
return { response: await page.goto(url.toString(), {
|
|
1399
|
+
waitUntil: "domcontentloaded",
|
|
1400
|
+
timeout: deadline.remainingMs()
|
|
1401
|
+
}) };
|
|
1402
|
+
} catch (error) {
|
|
1403
|
+
if (!deadline.signal.aborted) {
|
|
1404
|
+
const recovered = await recoverUnrenderedStatus(error, tracker, url, denoise);
|
|
1405
|
+
if (recovered !== void 0) return {
|
|
1406
|
+
response: null,
|
|
1407
|
+
recovered
|
|
1408
|
+
};
|
|
1409
|
+
}
|
|
1410
|
+
throw error;
|
|
1411
|
+
}
|
|
1412
|
+
}
|
|
1413
|
+
/**
|
|
1380
1414
|
* Whether the freshly loaded document is a Cloudflare interstitial: the
|
|
1381
1415
|
* response-level signals first (the documented `cf-mitigated` header, then
|
|
1382
1416
|
* the 403/503 + cloudflare fallback), then — gated behind
|
|
@@ -1463,6 +1497,79 @@ function trackMainFrameResponses(page) {
|
|
|
1463
1497
|
};
|
|
1464
1498
|
}
|
|
1465
1499
|
/**
|
|
1500
|
+
* Whether a `goto` throw is Chromium refusing to RENDER a response it
|
|
1501
|
+
* received — not a transport failure. Two net errors, both documented as
|
|
1502
|
+
* rendering decisions rather than network ones:
|
|
1503
|
+
*
|
|
1504
|
+
* - `ERR_HTTP_RESPONSE_CODE_FAILURE` — an error status whose empty body
|
|
1505
|
+
* leaves nothing to commit (httpbin's `/status/N`); net_error_list.h
|
|
1506
|
+
* describes the code as "only used by certain APIs that interpret the
|
|
1507
|
+
* HTTP response itself".
|
|
1508
|
+
* - `ERR_ABORTED` on a status-only 204/205 — Chromium never renders those
|
|
1509
|
+
* documents (`NavigationRequest` aborts them), while the response is
|
|
1510
|
+
* final and the seam contract wants it as a result.
|
|
1511
|
+
*
|
|
1512
|
+
* @param error - the value `page.goto` rejected with.
|
|
1513
|
+
* @param status - the status of the last tracked main-frame response.
|
|
1514
|
+
* @returns true when the navigation failed for the status it received.
|
|
1515
|
+
*/
|
|
1516
|
+
function isUnrenderedStatusFailure(error, status) {
|
|
1517
|
+
if (!(error instanceof Error)) return false;
|
|
1518
|
+
const message = error.message;
|
|
1519
|
+
if (message.includes("ERR_HTTP_RESPONSE_CODE_FAILURE")) return true;
|
|
1520
|
+
return message.includes("ERR_ABORTED") && (status === 204 || status === 205);
|
|
1521
|
+
}
|
|
1522
|
+
/**
|
|
1523
|
+
* Recover the response behind a status-only navigation failure into the
|
|
1524
|
+
* result the web seam's contract demands — "a successful network fetch of a
|
|
1525
|
+
* non-2xx response is a result, not an error: the status code is part of the
|
|
1526
|
+
* fetched resource state" (`@deepseek-ai/dsh-web` types). The main-frame
|
|
1527
|
+
* response arrives (and the tracker records it) before Chromium fails the
|
|
1528
|
+
* navigation, so its status, headers, and URL are recoverable; only the body
|
|
1529
|
+
* dies with the unrendered loader, and this failure class only fires on
|
|
1530
|
+
* empty bodies — so an unreadable body decodes as the empty string it is.
|
|
1531
|
+
*
|
|
1532
|
+
* Deliberately NOT recovered: genuine transport errors (they carry no
|
|
1533
|
+
* response), and our own deadline/abort (the caller classifies those).
|
|
1534
|
+
*
|
|
1535
|
+
* @param error - the value `page.goto` rejected with.
|
|
1536
|
+
* @param tracker - the fetch's main-frame response record.
|
|
1537
|
+
* @param url - the request URL (fallback when the response carries none).
|
|
1538
|
+
* @param denoise - the settings toggle, honored as on the normal HTML path.
|
|
1539
|
+
* @returns the finished fetch result, or undefined when the failure is not
|
|
1540
|
+
* a recoverable status-only navigation (the throw should propagate).
|
|
1541
|
+
*/
|
|
1542
|
+
async function recoverUnrenderedStatus(error, tracker, url, denoise) {
|
|
1543
|
+
const response = tracker.last();
|
|
1544
|
+
if (response === null) return void 0;
|
|
1545
|
+
const status = response.status();
|
|
1546
|
+
if (!isUnrenderedStatusFailure(error, status)) return void 0;
|
|
1547
|
+
const headers = response.headers();
|
|
1548
|
+
if (classifyChallengeResponse(status, headers) === "challenge") throw new WebError(`the site answered with a Cloudflare challenge response (HTTP ${String(status)}) whose empty body the browser refused to render; waiting cannot clear it — retry later or use a profile whose browser already holds clearance`, WEB_FETCH_CHALLENGE_CODE);
|
|
1549
|
+
const kind = classifyContentType(headers["content-type"]);
|
|
1550
|
+
if (kind === void 0) throw new WebError(`unsupported content type "${headers["content-type"] ?? "unknown"}"`, "WEB_UNSUPPORTED_CONTENT_TYPE");
|
|
1551
|
+
const finalUrl = response.url?.() ?? url.toString();
|
|
1552
|
+
let content = "";
|
|
1553
|
+
try {
|
|
1554
|
+
content = await response.text();
|
|
1555
|
+
} catch {
|
|
1556
|
+
content = "";
|
|
1557
|
+
}
|
|
1558
|
+
if (kind === "text" || content === "") return capResult(finalUrl, status, {
|
|
1559
|
+
kind: "text",
|
|
1560
|
+
content
|
|
1561
|
+
});
|
|
1562
|
+
if (!denoise) return capResult(finalUrl, status, {
|
|
1563
|
+
kind: "html",
|
|
1564
|
+
content
|
|
1565
|
+
});
|
|
1566
|
+
const { markdown } = htmlToMarkdown(content, finalUrl);
|
|
1567
|
+
return capResult(finalUrl, status, {
|
|
1568
|
+
kind: "text",
|
|
1569
|
+
content: markdown
|
|
1570
|
+
});
|
|
1571
|
+
}
|
|
1572
|
+
/**
|
|
1466
1573
|
* Probe the live document for challenge markers. Prefers the in-page probe
|
|
1467
1574
|
* (`page.evaluate`) — the only thing that can see SPA-style clears — and
|
|
1468
1575
|
* falls back to reading and classifying the serialized HTML when the backend
|
package/lib/types/provider.d.ts
CHANGED
|
@@ -2,7 +2,16 @@
|
|
|
2
2
|
* The Playwright `WebFetchProvider`: renders one URL in a real browser and
|
|
3
3
|
* returns it as markdown (denoised) or HTML. Mirrors `dsh-web-fetch-http`'s
|
|
4
4
|
* error taxonomy (URL hygiene, abort/timeout translation, content-type
|
|
5
|
-
* classification) so the tool layer sees the same codes from either backend
|
|
5
|
+
* classification) so the tool layer sees the same codes from either backend —
|
|
6
|
+
* including the seam contract that a fetched non-2xx status is a RESULT, not
|
|
7
|
+
* an error. Chromium can make that awkward: it refuses to render status-only
|
|
8
|
+
* responses (error statuses with an empty body — httpbin's `/status/N` —
|
|
9
|
+
* plus 204/205), failing the whole navigation with
|
|
10
|
+
* `net::ERR_HTTP_RESPONSE_CODE_FAILURE` / `net::ERR_ABORTED` even though the
|
|
11
|
+
* response itself arrived. The last main-frame response is therefore tracked
|
|
12
|
+
* on every fetch, and such a failed `goto` is recovered into the
|
|
13
|
+
* status-bearing result the contract demands instead of a
|
|
14
|
+
* `WEB_PROVIDER_ERROR` (see {@link recoverUnrenderedStatus}).
|
|
6
15
|
*
|
|
7
16
|
* Lifecycle: the local backend launches a browser per fetch and closes it —
|
|
8
17
|
* nothing outlives the call. The CDP backend keeps ONE shared connection to
|
|
@@ -121,6 +130,17 @@ export declare class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
121
130
|
* budget before the final document is read (see the class docs).
|
|
122
131
|
*/
|
|
123
132
|
private retrieve;
|
|
133
|
+
/**
|
|
134
|
+
* One tracked `goto`: the navigation response on success, or — when
|
|
135
|
+
* Chromium failed the navigation purely because of the HTTP status it
|
|
136
|
+
* received (an error status with an empty body, or a status-only 204/205)
|
|
137
|
+
* — the recovered status-bearing result from {@link recoverUnrenderedStatus}.
|
|
138
|
+
*
|
|
139
|
+
* @returns the response `goto` resolved with, plus `recovered` when the
|
|
140
|
+
* navigation failure was recovered into a finished result the caller must
|
|
141
|
+
* return verbatim (the throw never escapes in that case).
|
|
142
|
+
*/
|
|
143
|
+
private navigate;
|
|
124
144
|
/**
|
|
125
145
|
* Whether the freshly loaded document is a Cloudflare interstitial: the
|
|
126
146
|
* response-level signals first (the documented `cf-mitigated` header, then
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dsh-web-fetch-playwright",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.10",
|
|
4
4
|
"description": "Playwright/CDP web-fetch provider for DeepSeek Harness: renders pages in a real browser, denoises them (Readability + DOMPurify), and returns markdown.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
package/src/provider.ts
CHANGED
|
@@ -2,7 +2,16 @@
|
|
|
2
2
|
* The Playwright `WebFetchProvider`: renders one URL in a real browser and
|
|
3
3
|
* returns it as markdown (denoised) or HTML. Mirrors `dsh-web-fetch-http`'s
|
|
4
4
|
* error taxonomy (URL hygiene, abort/timeout translation, content-type
|
|
5
|
-
* classification) so the tool layer sees the same codes from either backend
|
|
5
|
+
* classification) so the tool layer sees the same codes from either backend —
|
|
6
|
+
* including the seam contract that a fetched non-2xx status is a RESULT, not
|
|
7
|
+
* an error. Chromium can make that awkward: it refuses to render status-only
|
|
8
|
+
* responses (error statuses with an empty body — httpbin's `/status/N` —
|
|
9
|
+
* plus 204/205), failing the whole navigation with
|
|
10
|
+
* `net::ERR_HTTP_RESPONSE_CODE_FAILURE` / `net::ERR_ABORTED` even though the
|
|
11
|
+
* response itself arrived. The last main-frame response is therefore tracked
|
|
12
|
+
* on every fetch, and such a failed `goto` is recovered into the
|
|
13
|
+
* status-bearing result the contract demands instead of a
|
|
14
|
+
* `WEB_PROVIDER_ERROR` (see {@link recoverUnrenderedStatus}).
|
|
6
15
|
*
|
|
7
16
|
* Lifecycle: the local backend launches a browser per fetch and closes it —
|
|
8
17
|
* nothing outlives the call. The CDP backend keeps ONE shared connection to
|
|
@@ -414,18 +423,23 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
414
423
|
): Promise<WebFetchResult> {
|
|
415
424
|
const page = session.page
|
|
416
425
|
const challengeWaitMs = effectiveChallengeWaitMs(config)
|
|
426
|
+
// The tracker is always on: the challenge wait needs the LAST main-frame
|
|
427
|
+
// response (feature switch below only gates the waiting), and status
|
|
428
|
+
// recovery needs whatever response arrived before a failed goto.
|
|
429
|
+
const tracker = trackMainFrameResponses(page)
|
|
417
430
|
// Feature switch: 0 keeps the exact legacy (pre-0.2.5) behavior — the
|
|
418
431
|
// first response decides, no waiting — an escape hatch and the A/B
|
|
419
432
|
// baseline every test proves the bug against.
|
|
420
|
-
const
|
|
421
|
-
|
|
422
|
-
|
|
433
|
+
const first = await this.navigate(page, url, tracker, deadline, config.denoise)
|
|
434
|
+
if (first.recovered !== undefined) return first.recovered
|
|
435
|
+
let response = first.response
|
|
436
|
+
tracker.seed(response)
|
|
423
437
|
|
|
424
438
|
let challengeEntryResponse: PlaywrightResponse | null = null
|
|
425
439
|
if (challengeWaitMs > 0) {
|
|
426
440
|
let attemptsLeft = effectiveChallengeRetries(config) + 1
|
|
427
441
|
for (;;) {
|
|
428
|
-
const verdict = await this.verdictAfterLoad(page, tracker
|
|
442
|
+
const verdict = await this.verdictAfterLoad(page, tracker.last() ?? response)
|
|
429
443
|
if (verdict === 'blocked') {
|
|
430
444
|
throw new WebError(
|
|
431
445
|
`the site hard-blocked this fetch at its Cloudflare edge (waiting cannot clear it): ${page.url()}`,
|
|
@@ -433,7 +447,7 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
433
447
|
)
|
|
434
448
|
}
|
|
435
449
|
if (verdict !== 'challenge') break
|
|
436
|
-
challengeEntryResponse = tracker
|
|
450
|
+
challengeEntryResponse = tracker.last() ?? response
|
|
437
451
|
const cleared = await this.waitForChallengeClear(page, deadline, challengeWaitMs)
|
|
438
452
|
if (cleared) {
|
|
439
453
|
// The settled document may be the next round of a CHAINED
|
|
@@ -446,7 +460,7 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
446
460
|
if (!chained) break
|
|
447
461
|
}
|
|
448
462
|
if (--attemptsLeft <= 0) {
|
|
449
|
-
const lastStatus = (tracker
|
|
463
|
+
const lastStatus = (tracker.last() ?? response)?.status()
|
|
450
464
|
throw new WebError(
|
|
451
465
|
`the site kept serving a Cloudflare challenge (last status ${lastStatus === undefined ? 'unknown' : String(lastStatus)}) for up to ${String(challengeWaitMs)}ms across ${String(effectiveChallengeRetries(config) + 1)} attempt(s); the browser did not clear it naturally — retry later, raise challengeWaitMs, or use a profile whose browser already holds clearance`,
|
|
452
466
|
WEB_FETCH_CHALLENGE_CODE,
|
|
@@ -455,14 +469,16 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
455
469
|
// Same page, same context — after an expired window OR a chained
|
|
456
470
|
// round: any clearance cookies already earned stay in the jar for
|
|
457
471
|
// this one retry, then everything is torn down as usual.
|
|
458
|
-
|
|
459
|
-
|
|
472
|
+
const retry = await this.navigate(page, url, tracker, deadline, config.denoise)
|
|
473
|
+
if (retry.recovered !== undefined) return retry.recovered
|
|
474
|
+
response = retry.response
|
|
475
|
+
tracker.seed(response)
|
|
460
476
|
}
|
|
461
477
|
}
|
|
462
478
|
|
|
463
479
|
// Final document: the LAST main-frame response when the challenge wait
|
|
464
480
|
// ran (the real page reloads in), else the response goto returned.
|
|
465
|
-
const finalResponse = (challengeWaitMs > 0 ? tracker
|
|
481
|
+
const finalResponse = (challengeWaitMs > 0 ? tracker.last() : undefined) ?? response
|
|
466
482
|
const kind = classifyContentType(finalResponse?.headers()['content-type'])
|
|
467
483
|
if (kind === undefined) {
|
|
468
484
|
throw new WebError(`unsupported content type "${finalResponse?.headers()['content-type'] ?? 'unknown'}"`, 'WEB_UNSUPPORTED_CONTENT_TYPE')
|
|
@@ -497,6 +513,37 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
497
513
|
: result
|
|
498
514
|
}
|
|
499
515
|
|
|
516
|
+
/**
|
|
517
|
+
* One tracked `goto`: the navigation response on success, or — when
|
|
518
|
+
* Chromium failed the navigation purely because of the HTTP status it
|
|
519
|
+
* received (an error status with an empty body, or a status-only 204/205)
|
|
520
|
+
* — the recovered status-bearing result from {@link recoverUnrenderedStatus}.
|
|
521
|
+
*
|
|
522
|
+
* @returns the response `goto` resolved with, plus `recovered` when the
|
|
523
|
+
* navigation failure was recovered into a finished result the caller must
|
|
524
|
+
* return verbatim (the throw never escapes in that case).
|
|
525
|
+
*/
|
|
526
|
+
private async navigate(
|
|
527
|
+
page: PlaywrightPage,
|
|
528
|
+
url: URL,
|
|
529
|
+
tracker: MainFrameTracker,
|
|
530
|
+
deadline: Deadline,
|
|
531
|
+
denoise: boolean,
|
|
532
|
+
): Promise<{ response: PlaywrightResponse | null; recovered?: WebFetchResult }> {
|
|
533
|
+
try {
|
|
534
|
+
const response = await page.goto(url.toString(), { waitUntil: 'domcontentloaded', timeout: deadline.remainingMs() })
|
|
535
|
+
return { response }
|
|
536
|
+
} catch (error) {
|
|
537
|
+
// Our own deadline/abort closing the page mid-navigation is a
|
|
538
|
+
// cancellation (translateError classifies it) — never a status fact.
|
|
539
|
+
if (!deadline.signal.aborted) {
|
|
540
|
+
const recovered = await recoverUnrenderedStatus(error, tracker, url, denoise)
|
|
541
|
+
if (recovered !== undefined) return { response: null, recovered }
|
|
542
|
+
}
|
|
543
|
+
throw error
|
|
544
|
+
}
|
|
545
|
+
}
|
|
546
|
+
|
|
500
547
|
/**
|
|
501
548
|
* Whether the freshly loaded document is a Cloudflare interstitial: the
|
|
502
549
|
* response-level signals first (the documented `cf-mitigated` header, then
|
|
@@ -608,6 +655,88 @@ function trackMainFrameResponses(page: PlaywrightPage): MainFrameTracker {
|
|
|
608
655
|
}
|
|
609
656
|
}
|
|
610
657
|
|
|
658
|
+
/**
|
|
659
|
+
* Whether a `goto` throw is Chromium refusing to RENDER a response it
|
|
660
|
+
* received — not a transport failure. Two net errors, both documented as
|
|
661
|
+
* rendering decisions rather than network ones:
|
|
662
|
+
*
|
|
663
|
+
* - `ERR_HTTP_RESPONSE_CODE_FAILURE` — an error status whose empty body
|
|
664
|
+
* leaves nothing to commit (httpbin's `/status/N`); net_error_list.h
|
|
665
|
+
* describes the code as "only used by certain APIs that interpret the
|
|
666
|
+
* HTTP response itself".
|
|
667
|
+
* - `ERR_ABORTED` on a status-only 204/205 — Chromium never renders those
|
|
668
|
+
* documents (`NavigationRequest` aborts them), while the response is
|
|
669
|
+
* final and the seam contract wants it as a result.
|
|
670
|
+
*
|
|
671
|
+
* @param error - the value `page.goto` rejected with.
|
|
672
|
+
* @param status - the status of the last tracked main-frame response.
|
|
673
|
+
* @returns true when the navigation failed for the status it received.
|
|
674
|
+
*/
|
|
675
|
+
function isUnrenderedStatusFailure(error: unknown, status: number): boolean {
|
|
676
|
+
if (!(error instanceof Error)) return false
|
|
677
|
+
const message = error.message
|
|
678
|
+
if (message.includes('ERR_HTTP_RESPONSE_CODE_FAILURE')) return true
|
|
679
|
+
return message.includes('ERR_ABORTED') && (status === 204 || status === 205)
|
|
680
|
+
}
|
|
681
|
+
|
|
682
|
+
/**
|
|
683
|
+
* Recover the response behind a status-only navigation failure into the
|
|
684
|
+
* result the web seam's contract demands — "a successful network fetch of a
|
|
685
|
+
* non-2xx response is a result, not an error: the status code is part of the
|
|
686
|
+
* fetched resource state" (`@deepseek-ai/dsh-web` types). The main-frame
|
|
687
|
+
* response arrives (and the tracker records it) before Chromium fails the
|
|
688
|
+
* navigation, so its status, headers, and URL are recoverable; only the body
|
|
689
|
+
* dies with the unrendered loader, and this failure class only fires on
|
|
690
|
+
* empty bodies — so an unreadable body decodes as the empty string it is.
|
|
691
|
+
*
|
|
692
|
+
* Deliberately NOT recovered: genuine transport errors (they carry no
|
|
693
|
+
* response), and our own deadline/abort (the caller classifies those).
|
|
694
|
+
*
|
|
695
|
+
* @param error - the value `page.goto` rejected with.
|
|
696
|
+
* @param tracker - the fetch's main-frame response record.
|
|
697
|
+
* @param url - the request URL (fallback when the response carries none).
|
|
698
|
+
* @param denoise - the settings toggle, honored as on the normal HTML path.
|
|
699
|
+
* @returns the finished fetch result, or undefined when the failure is not
|
|
700
|
+
* a recoverable status-only navigation (the throw should propagate).
|
|
701
|
+
*/
|
|
702
|
+
async function recoverUnrenderedStatus(
|
|
703
|
+
error: unknown,
|
|
704
|
+
tracker: MainFrameTracker,
|
|
705
|
+
url: URL,
|
|
706
|
+
denoise: boolean,
|
|
707
|
+
): Promise<WebFetchResult | undefined> {
|
|
708
|
+
const response = tracker.last()
|
|
709
|
+
if (response === null) return undefined
|
|
710
|
+
const status = response.status()
|
|
711
|
+
if (!isUnrenderedStatusFailure(error, status)) return undefined
|
|
712
|
+
const headers = response.headers()
|
|
713
|
+
// A challenge edge answering with nothing to render cannot be waited out
|
|
714
|
+
// (no document ever loaded to clear); keep the plugin's invariant of
|
|
715
|
+
// never passing challenge responses through as content.
|
|
716
|
+
if (classifyChallengeResponse(status, headers) === 'challenge') {
|
|
717
|
+
throw new WebError(
|
|
718
|
+
`the site answered with a Cloudflare challenge response (HTTP ${String(status)}) whose empty body the browser refused to render; waiting cannot clear it — retry later or use a profile whose browser already holds clearance`,
|
|
719
|
+
WEB_FETCH_CHALLENGE_CODE,
|
|
720
|
+
)
|
|
721
|
+
}
|
|
722
|
+
const kind = classifyContentType(headers['content-type'])
|
|
723
|
+
if (kind === undefined) {
|
|
724
|
+
throw new WebError(`unsupported content type "${headers['content-type'] ?? 'unknown'}"`, 'WEB_UNSUPPORTED_CONTENT_TYPE')
|
|
725
|
+
}
|
|
726
|
+
const finalUrl = response.url?.() ?? url.toString()
|
|
727
|
+
// The unrenderable loader may refuse the body read too; this failure
|
|
728
|
+
// class only fires on empty bodies, so '' is the honest fallback.
|
|
729
|
+
let content = ''
|
|
730
|
+
try { content = await response.text() } catch { content = '' }
|
|
731
|
+
if (kind === 'text' || content === '') {
|
|
732
|
+
// Empty HTML needs no pipeline — the empty string IS the body.
|
|
733
|
+
return capResult(finalUrl, status, { kind: 'text', content })
|
|
734
|
+
}
|
|
735
|
+
if (!denoise) return capResult(finalUrl, status, { kind: 'html', content })
|
|
736
|
+
const { markdown } = htmlToMarkdown(content, finalUrl)
|
|
737
|
+
return capResult(finalUrl, status, { kind: 'text', content: markdown })
|
|
738
|
+
}
|
|
739
|
+
|
|
611
740
|
/**
|
|
612
741
|
* Probe the live document for challenge markers. Prefers the in-page probe
|
|
613
742
|
* (`page.evaluate`) — the only thing that can see SPA-style clears — and
|