dsh-web-fetch-playwright 0.2.9 → 0.2.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/README.zh-CN.md +2 -0
- package/lib/client.js +16 -16
- package/lib/index.js +237 -16
- package/lib/types/provider.d.ts +29 -1
- package/lib/types/types.d.ts +33 -0
- package/package.json +1 -1
- package/src/provider.ts +267 -11
- package/src/types.ts +32 -0
package/README.md
CHANGED
|
@@ -34,6 +34,8 @@ web_fetch (tool-web)
|
|
|
34
34
|
└─ Markdown (or raw HTML when denoise is off)
|
|
35
35
|
```
|
|
36
36
|
|
|
37
|
+
**Non-2xx statuses are results, not errors** — the web seam's contract (`@deepseek-ai/dsh-web`): a fetched 404/418/500 comes back with its status code, exactly what `dsh-web-fetch-http` does. Chromium complicates this for *status-only* responses — an error status with an empty body (httpbin's `/status/404`) fails the whole navigation with `net::ERR_HTTP_RESPONSE_CODE_FAILURE`, and a 204/205 is aborted (`net::ERR_ABORTED`) because nothing is renderable. The provider therefore tracks the last main-frame response on every fetch (that response arrives before Chromium fails the navigation) and recovers such failures into the status-bearing result, so `web_fetch` reports `Fetched … (HTTP 404)` instead of a `WEB_PROVIDER_ERROR` throw. The same recovery covers *download* responses — a body served with `Content-Disposition: attachment` (publish.twitter.com's oembed endpoint, the URL agents use to read a tweet, serves its JSON exactly that way) never commits a document: `goto` rejects with `Download is starting` and the body is gone from the network stack, so the provider refetches it through the page's API request context and returns the decodable content; undecodable downloads (a PDF, a ZIP) still fail with `WEB_UNSUPPORTED_CONTENT_TYPE`. A recovered response that is itself a Cloudflare challenge edge still fails with `WEB_FETCH_CHALLENGE`; transport errors and cancellations stay errors.
|
|
38
|
+
|
|
37
39
|
## Requirements
|
|
38
40
|
|
|
39
41
|
- DSH web profile (`dsh web`), Node.js ≥ 20.
|
package/README.zh-CN.md
CHANGED
|
@@ -34,6 +34,8 @@ web_fetch (tool-web)
|
|
|
34
34
|
└─ Markdown(关闭降噪时返回原始 HTML)
|
|
35
35
|
```
|
|
36
36
|
|
|
37
|
+
**非 2xx 状态码是结果而非错误** —— 这是 web seam 的契约(`@deepseek-ai/dsh-web`):抓取到的 404/418/500 会连同状态码一起正常返回,与 `dsh-web-fetch-http` 的行为一致。但对*仅有状态*的响应 Chromium 会把事情搞砸 —— 空 body 的错误状态(httpbin 的 `/status/404`)会让整个导航以 `net::ERR_HTTP_RESPONSE_CODE_FAILURE` 失败,204/205 则因无可渲染内容被中止(`net::ERR_ABORTED`)。因此 provider 在每次抓取中都跟踪最后一个主框架响应(该响应先于 Chromium 判定导航失败到达),并把这类失败恢复成带状态码的正常结果,`web_fetch` 会报告 `Fetched … (HTTP 404)` 而不是抛出 `WEB_PROVIDER_ERROR`。同样的恢复也覆盖*下载型*响应 —— 带 `Content-Disposition: attachment` 的 body(publish.twitter.com 的 oembed 端点——agent 读取推文常用的替代 URL——返回的 JSON 正是如此)永远不会被提交为文档:`goto` 以 `Download is starting` 拒绝,且 body 已从网络栈中流失,provider 因此通过页面的 API request context 重新拉取并返回可解码内容;不可解码的下载(PDF、ZIP)仍以 `WEB_UNSUPPORTED_CONTENT_TYPE` 失败。若恢复出的响应本身来自 Cloudflare 挑战边缘,仍以 `WEB_FETCH_CHALLENGE` 失败;传输错误与取消也依旧按错误处理。
|
|
38
|
+
|
|
37
39
|
## 环境要求
|
|
38
40
|
|
|
39
41
|
- DSH web profile(`dsh web`),Node.js ≥ 20。
|
package/lib/client.js
CHANGED
|
@@ -412,12 +412,12 @@ window.__ModuleLoader__.load({
|
|
|
412
412
|
document.head.appendChild(tag);
|
|
413
413
|
}
|
|
414
414
|
var PluginCard_module_css_default = {
|
|
415
|
+
"readOnly": "UM8-Rq_readOnly",
|
|
416
|
+
"failed": "UM8-Rq_failed",
|
|
415
417
|
"discard": "UM8-Rq_discard",
|
|
416
418
|
"save": "UM8-Rq_save",
|
|
417
419
|
"footer": "UM8-Rq_footer",
|
|
418
|
-
"
|
|
419
|
-
"form": "UM8-Rq_form",
|
|
420
|
-
"readOnly": "UM8-Rq_readOnly"
|
|
420
|
+
"form": "UM8-Rq_form"
|
|
421
421
|
};
|
|
422
422
|
//#endregion
|
|
423
423
|
//#region src/client/PluginCard.tsx
|
|
@@ -477,26 +477,26 @@ window.__ModuleLoader__.load({
|
|
|
477
477
|
document.head.appendChild(tag);
|
|
478
478
|
}
|
|
479
479
|
var fields_module_css_default = {
|
|
480
|
-
"checkboxRow": "CFSaVG_checkboxRow",
|
|
481
|
-
"radioGroup": "CFSaVG_radioGroup",
|
|
482
|
-
"fieldEmbedded": "CFSaVG_fieldEmbedded",
|
|
483
|
-
"radioContent": "CFSaVG_radioContent",
|
|
484
|
-
"radioOption": "CFSaVG_radioOption",
|
|
485
|
-
"inputInvalid": "CFSaVG_inputInvalid",
|
|
486
|
-
"radioText": "CFSaVG_radioText",
|
|
487
|
-
"badge": "CFSaVG_badge",
|
|
488
480
|
"radioLabel": "CFSaVG_radioLabel",
|
|
489
|
-
"
|
|
481
|
+
"radioHint": "CFSaVG_radioHint",
|
|
490
482
|
"input": "CFSaVG_input",
|
|
491
|
-
"
|
|
483
|
+
"radioText": "CFSaVG_radioText",
|
|
492
484
|
"invalid": "CFSaVG_invalid",
|
|
493
|
-
"
|
|
485
|
+
"radioGroup": "CFSaVG_radioGroup",
|
|
486
|
+
"radioContent": "CFSaVG_radioContent",
|
|
494
487
|
"label": "CFSaVG_label",
|
|
488
|
+
"field": "CFSaVG_field",
|
|
489
|
+
"hint": "CFSaVG_hint",
|
|
490
|
+
"fieldEmbedded": "CFSaVG_fieldEmbedded",
|
|
491
|
+
"badges": "CFSaVG_badges",
|
|
495
492
|
"reset": "CFSaVG_reset",
|
|
493
|
+
"head": "CFSaVG_head",
|
|
494
|
+
"radioOption": "CFSaVG_radioOption",
|
|
495
|
+
"inputInvalid": "CFSaVG_inputInvalid",
|
|
496
496
|
"hintIndented": "CFSaVG_hintIndented",
|
|
497
|
+
"badge": "CFSaVG_badge",
|
|
497
498
|
"radioPick": "CFSaVG_radioPick",
|
|
498
|
-
"
|
|
499
|
-
"hint": "CFSaVG_hint"
|
|
499
|
+
"checkboxRow": "CFSaVG_checkboxRow"
|
|
500
500
|
};
|
|
501
501
|
//#endregion
|
|
502
502
|
//#region src/client/fields.tsx
|
package/lib/index.js
CHANGED
|
@@ -993,7 +993,21 @@ function existsAsFile(path) {
|
|
|
993
993
|
* The Playwright `WebFetchProvider`: renders one URL in a real browser and
|
|
994
994
|
* returns it as markdown (denoised) or HTML. Mirrors `dsh-web-fetch-http`'s
|
|
995
995
|
* error taxonomy (URL hygiene, abort/timeout translation, content-type
|
|
996
|
-
* classification) so the tool layer sees the same codes from either backend
|
|
996
|
+
* classification) so the tool layer sees the same codes from either backend —
|
|
997
|
+
* including the seam contract that a fetched non-2xx status is a RESULT, not
|
|
998
|
+
* an error. Chromium can make that awkward: it refuses to render status-only
|
|
999
|
+
* responses (error statuses with an empty body — httpbin's `/status/N` —
|
|
1000
|
+
* plus 204/205), failing the whole navigation with
|
|
1001
|
+
* `net::ERR_HTTP_RESPONSE_CODE_FAILURE` / `net::ERR_ABORTED`, and it hands a
|
|
1002
|
+
* `Content-Disposition: attachment` response (publish.twitter.com's oembed
|
|
1003
|
+
* JSON, the URL agents use to read a tweet) to its download manager instead
|
|
1004
|
+
* of committing a document, failing `goto` with "Download is starting" —
|
|
1005
|
+
* even though the response itself arrived and is decodable. The last
|
|
1006
|
+
* main-frame response is therefore tracked on every fetch, and such failed
|
|
1007
|
+
* `goto`s are recovered — the status-only shape into the status-bearing
|
|
1008
|
+
* result ({@link recoverUnrenderedStatus}), the download shape via a
|
|
1009
|
+
* refetch through the page's API request context
|
|
1010
|
+
* ({@link recoverDownloadResponse}) — instead of a `WEB_PROVIDER_ERROR`.
|
|
997
1011
|
*
|
|
998
1012
|
* Lifecycle: the local backend launches a browser per fetch and closes it —
|
|
999
1013
|
* nothing outlives the call. The CDP backend keeps ONE shared connection to
|
|
@@ -1275,6 +1289,7 @@ var PlaywrightFetchProvider = class {
|
|
|
1275
1289
|
const lease = await this.cdpPool.acquire(endpoint, timeout, effectiveContextMode(config));
|
|
1276
1290
|
await installResourceFilter(lease.page);
|
|
1277
1291
|
guardPopups(lease.page);
|
|
1292
|
+
guardDownloads(lease.page);
|
|
1278
1293
|
return {
|
|
1279
1294
|
browser: lease.browser,
|
|
1280
1295
|
context: lease.context,
|
|
@@ -1298,6 +1313,7 @@ var PlaywrightFetchProvider = class {
|
|
|
1298
1313
|
const page = await context.newPage();
|
|
1299
1314
|
await installResourceFilter(page);
|
|
1300
1315
|
guardPopups(page);
|
|
1316
|
+
guardDownloads(page);
|
|
1301
1317
|
return {
|
|
1302
1318
|
browser,
|
|
1303
1319
|
context,
|
|
@@ -1316,20 +1332,19 @@ var PlaywrightFetchProvider = class {
|
|
|
1316
1332
|
async retrieve(session, url, config, deadline) {
|
|
1317
1333
|
const page = session.page;
|
|
1318
1334
|
const challengeWaitMs = effectiveChallengeWaitMs(config);
|
|
1319
|
-
const tracker =
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
tracker?.seed(response);
|
|
1335
|
+
const tracker = trackMainFrameResponses(page);
|
|
1336
|
+
const first = await this.navigate(page, url, tracker, deadline, config.denoise);
|
|
1337
|
+
if (first.recovered !== void 0) return first.recovered;
|
|
1338
|
+
let response = first.response;
|
|
1339
|
+
tracker.seed(response);
|
|
1325
1340
|
let challengeEntryResponse = null;
|
|
1326
1341
|
if (challengeWaitMs > 0) {
|
|
1327
1342
|
let attemptsLeft = effectiveChallengeRetries(config) + 1;
|
|
1328
1343
|
for (;;) {
|
|
1329
|
-
const verdict = await this.verdictAfterLoad(page, tracker
|
|
1344
|
+
const verdict = await this.verdictAfterLoad(page, tracker.last() ?? response);
|
|
1330
1345
|
if (verdict === "blocked") throw new WebError(`the site hard-blocked this fetch at its Cloudflare edge (waiting cannot clear it): ${page.url()}`, WEB_FETCH_CHALLENGE_CODE);
|
|
1331
1346
|
if (verdict !== "challenge") break;
|
|
1332
|
-
challengeEntryResponse = tracker
|
|
1347
|
+
challengeEntryResponse = tracker.last() ?? response;
|
|
1333
1348
|
if (await this.waitForChallengeClear(page, deadline, challengeWaitMs)) {
|
|
1334
1349
|
let chained = false;
|
|
1335
1350
|
try {
|
|
@@ -1340,17 +1355,16 @@ var PlaywrightFetchProvider = class {
|
|
|
1340
1355
|
if (!chained) break;
|
|
1341
1356
|
}
|
|
1342
1357
|
if (--attemptsLeft <= 0) {
|
|
1343
|
-
const lastStatus = (tracker
|
|
1358
|
+
const lastStatus = (tracker.last() ?? response)?.status();
|
|
1344
1359
|
throw new WebError(`the site kept serving a Cloudflare challenge (last status ${lastStatus === void 0 ? "unknown" : String(lastStatus)}) for up to ${String(challengeWaitMs)}ms across ${String(effectiveChallengeRetries(config) + 1)} attempt(s); the browser did not clear it naturally — retry later, raise challengeWaitMs, or use a profile whose browser already holds clearance`, WEB_FETCH_CHALLENGE_CODE);
|
|
1345
1360
|
}
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
tracker?.seed(response);
|
|
1361
|
+
const retry = await this.navigate(page, url, tracker, deadline, config.denoise);
|
|
1362
|
+
if (retry.recovered !== void 0) return retry.recovered;
|
|
1363
|
+
response = retry.response;
|
|
1364
|
+
tracker.seed(response);
|
|
1351
1365
|
}
|
|
1352
1366
|
}
|
|
1353
|
-
const finalResponse = (challengeWaitMs > 0 ? tracker
|
|
1367
|
+
const finalResponse = (challengeWaitMs > 0 ? tracker.last() : void 0) ?? response;
|
|
1354
1368
|
const kind = classifyContentType(finalResponse?.headers()["content-type"]);
|
|
1355
1369
|
if (kind === void 0) throw new WebError(`unsupported content type "${finalResponse?.headers()["content-type"] ?? "unknown"}"`, "WEB_UNSUPPORTED_CONTENT_TYPE");
|
|
1356
1370
|
const finalUrl = page.url();
|
|
@@ -1377,6 +1391,41 @@ var PlaywrightFetchProvider = class {
|
|
|
1377
1391
|
} : result;
|
|
1378
1392
|
}
|
|
1379
1393
|
/**
|
|
1394
|
+
* One tracked `goto`: the navigation response on success, or — when
|
|
1395
|
+
* Chromium failed the navigation for a reason that is not a transport
|
|
1396
|
+
* error — the recovered finished result. Two such reasons are recovered:
|
|
1397
|
+
* the HTTP status it received (an error status with an empty body, or a
|
|
1398
|
+
* status-only 204/205 — {@link recoverUnrenderedStatus}), and the response
|
|
1399
|
+
* being a download the browser refuses to commit as a document
|
|
1400
|
+
* (`Content-Disposition: attachment` — {@link recoverDownloadResponse}).
|
|
1401
|
+
*
|
|
1402
|
+
* @returns the response `goto` resolved with, plus `recovered` when the
|
|
1403
|
+
* navigation failure was recovered into a finished result the caller must
|
|
1404
|
+
* return verbatim (the throw never escapes in that case).
|
|
1405
|
+
*/
|
|
1406
|
+
async navigate(page, url, tracker, deadline, denoise) {
|
|
1407
|
+
try {
|
|
1408
|
+
return { response: await page.goto(url.toString(), {
|
|
1409
|
+
waitUntil: "domcontentloaded",
|
|
1410
|
+
timeout: deadline.remainingMs()
|
|
1411
|
+
}) };
|
|
1412
|
+
} catch (error) {
|
|
1413
|
+
if (!deadline.signal.aborted) {
|
|
1414
|
+
const recovered = await recoverUnrenderedStatus(error, tracker, url, denoise);
|
|
1415
|
+
if (recovered !== void 0) return {
|
|
1416
|
+
response: null,
|
|
1417
|
+
recovered
|
|
1418
|
+
};
|
|
1419
|
+
const download = await recoverDownloadResponse(error, page, tracker, url, deadline, denoise);
|
|
1420
|
+
if (download !== void 0) return {
|
|
1421
|
+
response: null,
|
|
1422
|
+
recovered: download
|
|
1423
|
+
};
|
|
1424
|
+
}
|
|
1425
|
+
throw error;
|
|
1426
|
+
}
|
|
1427
|
+
}
|
|
1428
|
+
/**
|
|
1380
1429
|
* Whether the freshly loaded document is a Cloudflare interstitial: the
|
|
1381
1430
|
* response-level signals first (the documented `cf-mitigated` header, then
|
|
1382
1431
|
* the 403/503 + cloudflare fallback), then — gated behind
|
|
@@ -1463,6 +1512,162 @@ function trackMainFrameResponses(page) {
|
|
|
1463
1512
|
};
|
|
1464
1513
|
}
|
|
1465
1514
|
/**
|
|
1515
|
+
* Whether a `goto` throw is Chromium refusing to RENDER a response it
|
|
1516
|
+
* received — not a transport failure. Two net errors, both documented as
|
|
1517
|
+
* rendering decisions rather than network ones:
|
|
1518
|
+
*
|
|
1519
|
+
* - `ERR_HTTP_RESPONSE_CODE_FAILURE` — an error status whose empty body
|
|
1520
|
+
* leaves nothing to commit (httpbin's `/status/N`); net_error_list.h
|
|
1521
|
+
* describes the code as "only used by certain APIs that interpret the
|
|
1522
|
+
* HTTP response itself".
|
|
1523
|
+
* - `ERR_ABORTED` on a status-only 204/205 — Chromium never renders those
|
|
1524
|
+
* documents (`NavigationRequest` aborts them), while the response is
|
|
1525
|
+
* final and the seam contract wants it as a result.
|
|
1526
|
+
*
|
|
1527
|
+
* @param error - the value `page.goto` rejected with.
|
|
1528
|
+
* @param status - the status of the last tracked main-frame response.
|
|
1529
|
+
* @returns true when the navigation failed for the status it received.
|
|
1530
|
+
*/
|
|
1531
|
+
function isUnrenderedStatusFailure(error, status) {
|
|
1532
|
+
if (!(error instanceof Error)) return false;
|
|
1533
|
+
const message = error.message;
|
|
1534
|
+
if (message.includes("ERR_HTTP_RESPONSE_CODE_FAILURE")) return true;
|
|
1535
|
+
return message.includes("ERR_ABORTED") && (status === 204 || status === 205);
|
|
1536
|
+
}
|
|
1537
|
+
/**
|
|
1538
|
+
* Recover the response behind a status-only navigation failure into the
|
|
1539
|
+
* result the web seam's contract demands — "a successful network fetch of a
|
|
1540
|
+
* non-2xx response is a result, not an error: the status code is part of the
|
|
1541
|
+
* fetched resource state" (`@deepseek-ai/dsh-web` types). The main-frame
|
|
1542
|
+
* response arrives (and the tracker records it) before Chromium fails the
|
|
1543
|
+
* navigation, so its status, headers, and URL are recoverable; only the body
|
|
1544
|
+
* dies with the unrendered loader, and this failure class only fires on
|
|
1545
|
+
* empty bodies — so an unreadable body decodes as the empty string it is.
|
|
1546
|
+
*
|
|
1547
|
+
* Deliberately NOT recovered: genuine transport errors (they carry no
|
|
1548
|
+
* response), and our own deadline/abort (the caller classifies those).
|
|
1549
|
+
*
|
|
1550
|
+
* @param error - the value `page.goto` rejected with.
|
|
1551
|
+
* @param tracker - the fetch's main-frame response record.
|
|
1552
|
+
* @param url - the request URL (fallback when the response carries none).
|
|
1553
|
+
* @param denoise - the settings toggle, honored as on the normal HTML path.
|
|
1554
|
+
* @returns the finished fetch result, or undefined when the failure is not
|
|
1555
|
+
* a recoverable status-only navigation (the throw should propagate).
|
|
1556
|
+
*/
|
|
1557
|
+
async function recoverUnrenderedStatus(error, tracker, url, denoise) {
|
|
1558
|
+
const response = tracker.last();
|
|
1559
|
+
if (response === null) return void 0;
|
|
1560
|
+
const status = response.status();
|
|
1561
|
+
if (!isUnrenderedStatusFailure(error, status)) return void 0;
|
|
1562
|
+
const headers = response.headers();
|
|
1563
|
+
if (classifyChallengeResponse(status, headers) === "challenge") throw new WebError(`the site answered with a Cloudflare challenge response (HTTP ${String(status)}) whose empty body the browser refused to render; waiting cannot clear it — retry later or use a profile whose browser already holds clearance`, WEB_FETCH_CHALLENGE_CODE);
|
|
1564
|
+
const kind = classifyContentType(headers["content-type"]);
|
|
1565
|
+
if (kind === void 0) throw new WebError(`unsupported content type "${headers["content-type"] ?? "unknown"}"`, "WEB_UNSUPPORTED_CONTENT_TYPE");
|
|
1566
|
+
const finalUrl = response.url?.() ?? url.toString();
|
|
1567
|
+
let content = "";
|
|
1568
|
+
try {
|
|
1569
|
+
content = await response.text();
|
|
1570
|
+
} catch {
|
|
1571
|
+
content = "";
|
|
1572
|
+
}
|
|
1573
|
+
if (kind === "text" || content === "") return capResult(finalUrl, status, {
|
|
1574
|
+
kind: "text",
|
|
1575
|
+
content
|
|
1576
|
+
});
|
|
1577
|
+
if (!denoise) return capResult(finalUrl, status, {
|
|
1578
|
+
kind: "html",
|
|
1579
|
+
content
|
|
1580
|
+
});
|
|
1581
|
+
const { markdown } = htmlToMarkdown(content, finalUrl);
|
|
1582
|
+
return capResult(finalUrl, status, {
|
|
1583
|
+
kind: "text",
|
|
1584
|
+
content: markdown
|
|
1585
|
+
});
|
|
1586
|
+
}
|
|
1587
|
+
/**
|
|
1588
|
+
* Whether a `goto` throw is Chromium handing the navigation to its download
|
|
1589
|
+
* manager instead of committing a document. Playwright rejects with
|
|
1590
|
+
* "Download is starting" when the navigation response carries
|
|
1591
|
+
* `Content-Disposition: attachment` (publish.twitter.com's oembed endpoint —
|
|
1592
|
+
* the workaround URL agents use to read a tweet — serves exactly that:
|
|
1593
|
+
* decodable `application/json` the browser still refuses to render). The
|
|
1594
|
+
* attachment disposition on the tracked response is a second,
|
|
1595
|
+
* engine-version-proof signal for engines that report a plain `ERR_ABORTED`
|
|
1596
|
+
* instead; note `ERR_ABORTED` alone is NOT enough (the status recovery
|
|
1597
|
+
* above legitimately owns the 204/205 shape, and SPA cancels abort too).
|
|
1598
|
+
*
|
|
1599
|
+
* @param error - the value `page.goto` rejected with.
|
|
1600
|
+
* @param response - the last tracked main-frame response, if any.
|
|
1601
|
+
* @returns true when the navigation failed because it became a download.
|
|
1602
|
+
*/
|
|
1603
|
+
function isDownloadFailure(error, response) {
|
|
1604
|
+
if (!(error instanceof Error)) return false;
|
|
1605
|
+
if (error.message.includes("Download is starting")) return true;
|
|
1606
|
+
if (!error.message.includes("ERR_ABORTED")) return false;
|
|
1607
|
+
return (response?.headers()["content-disposition"] ?? "").toLowerCase().includes("attachment");
|
|
1608
|
+
}
|
|
1609
|
+
/**
|
|
1610
|
+
* Recover a download-turned navigation into the finished fetch result the
|
|
1611
|
+
* seam's contract demands. The resource itself is often perfectly decodable
|
|
1612
|
+
* — the oembed endpoint's `application/json` classifies as `text` — the
|
|
1613
|
+
* browser just will not commit it as a document, and the response it tracked
|
|
1614
|
+
* cannot yield its body either (Chromium streams a download to its download
|
|
1615
|
+
* manager, so `Network.getResponseBody` answers "No resource with given
|
|
1616
|
+
* identifier found"; both verified against a real browser). The body is
|
|
1617
|
+
* therefore refetched through the page's API request context: plain HTTP
|
|
1618
|
+
* from the Playwright client — same cookie jar, same URL, no render-or-
|
|
1619
|
+
* download decision — which works identically on the local and CDP
|
|
1620
|
+
* backends because it never touches the remote artifact.
|
|
1621
|
+
*
|
|
1622
|
+
* Undecodable downloads (a PDF, a ZIP) still fail with
|
|
1623
|
+
* `WEB_UNSUPPORTED_CONTENT_TYPE`, classified from what the navigation
|
|
1624
|
+
* actually served; a playwright build without a request context surfaces a
|
|
1625
|
+
* clear "the browser will not render this" error instead of the raw
|
|
1626
|
+
* "Download is starting".
|
|
1627
|
+
*
|
|
1628
|
+
* @param error - the value `page.goto` rejected with.
|
|
1629
|
+
* @param page - the fetch's tab, for its API request context.
|
|
1630
|
+
* @param tracker - the fetch's main-frame response record.
|
|
1631
|
+
* @param url - the request URL (fallback when the response carries none).
|
|
1632
|
+
* @param deadline - the fetch budget, applied to the refetch.
|
|
1633
|
+
* @param denoise - the settings toggle, honored as on the normal HTML path.
|
|
1634
|
+
* @returns the finished fetch result, or undefined when the failure is not
|
|
1635
|
+
* a download navigation (the throw should propagate).
|
|
1636
|
+
*/
|
|
1637
|
+
async function recoverDownloadResponse(error, page, tracker, url, deadline, denoise) {
|
|
1638
|
+
const tracked = tracker.last();
|
|
1639
|
+
if (!isDownloadFailure(error, tracked)) return void 0;
|
|
1640
|
+
if (tracked !== null) {
|
|
1641
|
+
if (classifyContentType(tracked.headers()["content-type"]) === void 0) throw new WebError(`unsupported content type "${tracked.headers()["content-type"] ?? "unknown"}" (served as a download)`, "WEB_UNSUPPORTED_CONTENT_TYPE");
|
|
1642
|
+
}
|
|
1643
|
+
if (page.request === void 0) throw new WebError("the response is a file download (Content-Disposition: attachment) the browser will not render, and this playwright build exposes no request context to refetch it", "WEB_PROVIDER_ERROR", { cause: error });
|
|
1644
|
+
const fetchUrl = tracked?.url?.() ?? url.toString();
|
|
1645
|
+
const refetched = await page.request.get(fetchUrl, {
|
|
1646
|
+
timeout: deadline.remainingMs(),
|
|
1647
|
+
signal: deadline.signal
|
|
1648
|
+
});
|
|
1649
|
+
const headers = refetched.headers();
|
|
1650
|
+
const servedType = (headers["content-type"] ?? "").trim() !== "" ? headers["content-type"] : tracked?.headers()["content-type"];
|
|
1651
|
+
const kind = classifyContentType(servedType);
|
|
1652
|
+
if (kind === void 0) throw new WebError(`unsupported content type "${servedType ?? "unknown"}"`, "WEB_UNSUPPORTED_CONTENT_TYPE");
|
|
1653
|
+
const finalUrl = refetched.url?.() ?? fetchUrl;
|
|
1654
|
+
const statusCode = refetched.status();
|
|
1655
|
+
const content = await refetched.text();
|
|
1656
|
+
if (kind === "text") return capResult(finalUrl, statusCode, {
|
|
1657
|
+
kind: "text",
|
|
1658
|
+
content
|
|
1659
|
+
});
|
|
1660
|
+
if (!denoise) return capResult(finalUrl, statusCode, {
|
|
1661
|
+
kind: "html",
|
|
1662
|
+
content
|
|
1663
|
+
});
|
|
1664
|
+
const { markdown } = htmlToMarkdown(content, finalUrl);
|
|
1665
|
+
return capResult(finalUrl, statusCode, {
|
|
1666
|
+
kind: "text",
|
|
1667
|
+
content: markdown
|
|
1668
|
+
});
|
|
1669
|
+
}
|
|
1670
|
+
/**
|
|
1466
1671
|
* Probe the live document for challenge markers. Prefers the in-page probe
|
|
1467
1672
|
* (`page.evaluate`) — the only thing that can see SPA-style clears — and
|
|
1468
1673
|
* falls back to reading and classifying the serialized HTML when the backend
|
|
@@ -1547,6 +1752,22 @@ async function installResourceFilter(owner) {
|
|
|
1547
1752
|
} catch {}
|
|
1548
1753
|
}
|
|
1549
1754
|
/**
|
|
1755
|
+
* Cancel any download Chromium starts on the fetch's tab. A navigation whose
|
|
1756
|
+
* response carries `Content-Disposition: attachment` never commits a
|
|
1757
|
+
* document — `goto` rejects with "Download is starting" and the recovery
|
|
1758
|
+
* refetches the bytes (see {@link recoverDownloadResponse}) — but the
|
|
1759
|
+
* browser still hands the original body to its download manager, which in
|
|
1760
|
+
* profile mode would drop files into the remote machine's download dir.
|
|
1761
|
+
* Best-effort: a page without the listener just loses the guard.
|
|
1762
|
+
*/
|
|
1763
|
+
function guardDownloads(page) {
|
|
1764
|
+
try {
|
|
1765
|
+
page.on?.("download", (download) => {
|
|
1766
|
+
download.cancel().catch(() => {});
|
|
1767
|
+
});
|
|
1768
|
+
} catch {}
|
|
1769
|
+
}
|
|
1770
|
+
/**
|
|
1550
1771
|
* Close any popup a fetched page spawns so nothing outlives the fetch's tab
|
|
1551
1772
|
* — `page.close()` does not auto-close windows the page opened, and in
|
|
1552
1773
|
* profile mode a stray tab would stay in the user's remote browser.
|
package/lib/types/provider.d.ts
CHANGED
|
@@ -2,7 +2,21 @@
|
|
|
2
2
|
* The Playwright `WebFetchProvider`: renders one URL in a real browser and
|
|
3
3
|
* returns it as markdown (denoised) or HTML. Mirrors `dsh-web-fetch-http`'s
|
|
4
4
|
* error taxonomy (URL hygiene, abort/timeout translation, content-type
|
|
5
|
-
* classification) so the tool layer sees the same codes from either backend
|
|
5
|
+
* classification) so the tool layer sees the same codes from either backend —
|
|
6
|
+
* including the seam contract that a fetched non-2xx status is a RESULT, not
|
|
7
|
+
* an error. Chromium can make that awkward: it refuses to render status-only
|
|
8
|
+
* responses (error statuses with an empty body — httpbin's `/status/N` —
|
|
9
|
+
* plus 204/205), failing the whole navigation with
|
|
10
|
+
* `net::ERR_HTTP_RESPONSE_CODE_FAILURE` / `net::ERR_ABORTED`, and it hands a
|
|
11
|
+
* `Content-Disposition: attachment` response (publish.twitter.com's oembed
|
|
12
|
+
* JSON, the URL agents use to read a tweet) to its download manager instead
|
|
13
|
+
* of committing a document, failing `goto` with "Download is starting" —
|
|
14
|
+
* even though the response itself arrived and is decodable. The last
|
|
15
|
+
* main-frame response is therefore tracked on every fetch, and such failed
|
|
16
|
+
* `goto`s are recovered — the status-only shape into the status-bearing
|
|
17
|
+
* result ({@link recoverUnrenderedStatus}), the download shape via a
|
|
18
|
+
* refetch through the page's API request context
|
|
19
|
+
* ({@link recoverDownloadResponse}) — instead of a `WEB_PROVIDER_ERROR`.
|
|
6
20
|
*
|
|
7
21
|
* Lifecycle: the local backend launches a browser per fetch and closes it —
|
|
8
22
|
* nothing outlives the call. The CDP backend keeps ONE shared connection to
|
|
@@ -121,6 +135,20 @@ export declare class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
121
135
|
* budget before the final document is read (see the class docs).
|
|
122
136
|
*/
|
|
123
137
|
private retrieve;
|
|
138
|
+
/**
|
|
139
|
+
* One tracked `goto`: the navigation response on success, or — when
|
|
140
|
+
* Chromium failed the navigation for a reason that is not a transport
|
|
141
|
+
* error — the recovered finished result. Two such reasons are recovered:
|
|
142
|
+
* the HTTP status it received (an error status with an empty body, or a
|
|
143
|
+
* status-only 204/205 — {@link recoverUnrenderedStatus}), and the response
|
|
144
|
+
* being a download the browser refuses to commit as a document
|
|
145
|
+
* (`Content-Disposition: attachment` — {@link recoverDownloadResponse}).
|
|
146
|
+
*
|
|
147
|
+
* @returns the response `goto` resolved with, plus `recovered` when the
|
|
148
|
+
* navigation failure was recovered into a finished result the caller must
|
|
149
|
+
* return verbatim (the throw never escapes in that case).
|
|
150
|
+
*/
|
|
151
|
+
private navigate;
|
|
124
152
|
/**
|
|
125
153
|
* Whether the freshly loaded document is a Cloudflare interstitial: the
|
|
126
154
|
* response-level signals first (the documented `cf-mitigated` header, then
|
package/lib/types/types.d.ts
CHANGED
|
@@ -20,6 +20,27 @@ export interface PlaywrightResponse {
|
|
|
20
20
|
*/
|
|
21
21
|
request?(): PlaywrightRequest;
|
|
22
22
|
}
|
|
23
|
+
/**
|
|
24
|
+
* A download Chromium started instead of committing a document (a response
|
|
25
|
+
* with `Content-Disposition: attachment`).
|
|
26
|
+
*/
|
|
27
|
+
export interface PlaywrightDownload {
|
|
28
|
+
cancel(): Promise<void>;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* The page's `APIRequestContext` surface the download recovery refetches
|
|
32
|
+
* through — plain HTTP from the Playwright client, the context's cookie jar,
|
|
33
|
+
* and none of Chromium's render-or-download semantics. Reuses
|
|
34
|
+
* {@link PlaywrightResponse} because that is structurally what an
|
|
35
|
+
* `APIResponse` is for the members this plugin reads. Absent on fakes that
|
|
36
|
+
* do not exercise the recovery.
|
|
37
|
+
*/
|
|
38
|
+
export interface PlaywrightRequestContext {
|
|
39
|
+
get(url: string, options?: {
|
|
40
|
+
timeout?: number;
|
|
41
|
+
signal?: AbortSignal;
|
|
42
|
+
}): Promise<PlaywrightResponse>;
|
|
43
|
+
}
|
|
23
44
|
/** The request side of a response, for main-frame filtering. */
|
|
24
45
|
export interface PlaywrightRequest {
|
|
25
46
|
/** True for navigations (document loads and their redirect hops). */
|
|
@@ -54,6 +75,13 @@ export interface PlaywrightPage {
|
|
|
54
75
|
* document). Absent on minimal fakes (content polling covers them).
|
|
55
76
|
*/
|
|
56
77
|
on?(event: 'response', listener: (response: PlaywrightResponse) => void): unknown;
|
|
78
|
+
/**
|
|
79
|
+
* Download notification — a navigation the browser handed to its download
|
|
80
|
+
* manager instead of committing a document (`Content-Disposition:
|
|
81
|
+
* attachment`); the fetch's guard cancels it so nothing lands on disk.
|
|
82
|
+
* Absent on minimal fakes (the recovery does not depend on it).
|
|
83
|
+
*/
|
|
84
|
+
on?(event: 'download', listener: (download: PlaywrightDownload) => void): unknown;
|
|
57
85
|
/**
|
|
58
86
|
* Evaluate an expression in the page — the challenge probe's live-DOM
|
|
59
87
|
* path. Absent on minimal fakes (content polling covers them).
|
|
@@ -61,6 +89,11 @@ export interface PlaywrightPage {
|
|
|
61
89
|
evaluate?(script: string, arg?: unknown): Promise<unknown>;
|
|
62
90
|
/** The main frame handle; compare with `request.frame()` for filtering. */
|
|
63
91
|
mainFrame?(): unknown;
|
|
92
|
+
/**
|
|
93
|
+
* The page's API request context — the download recovery's refetch path
|
|
94
|
+
* (see {@link PlaywrightRequestContext}). Absent on minimal fakes.
|
|
95
|
+
*/
|
|
96
|
+
request?: PlaywrightRequestContext;
|
|
64
97
|
}
|
|
65
98
|
/**
|
|
66
99
|
* A browser context: fetch-owned and isolated (local backend, or CDP
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dsh-web-fetch-playwright",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.11",
|
|
4
4
|
"description": "Playwright/CDP web-fetch provider for DeepSeek Harness: renders pages in a real browser, denoises them (Readability + DOMPurify), and returns markdown.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
package/src/provider.ts
CHANGED
|
@@ -2,7 +2,21 @@
|
|
|
2
2
|
* The Playwright `WebFetchProvider`: renders one URL in a real browser and
|
|
3
3
|
* returns it as markdown (denoised) or HTML. Mirrors `dsh-web-fetch-http`'s
|
|
4
4
|
* error taxonomy (URL hygiene, abort/timeout translation, content-type
|
|
5
|
-
* classification) so the tool layer sees the same codes from either backend
|
|
5
|
+
* classification) so the tool layer sees the same codes from either backend —
|
|
6
|
+
* including the seam contract that a fetched non-2xx status is a RESULT, not
|
|
7
|
+
* an error. Chromium can make that awkward: it refuses to render status-only
|
|
8
|
+
* responses (error statuses with an empty body — httpbin's `/status/N` —
|
|
9
|
+
* plus 204/205), failing the whole navigation with
|
|
10
|
+
* `net::ERR_HTTP_RESPONSE_CODE_FAILURE` / `net::ERR_ABORTED`, and it hands a
|
|
11
|
+
* `Content-Disposition: attachment` response (publish.twitter.com's oembed
|
|
12
|
+
* JSON, the URL agents use to read a tweet) to its download manager instead
|
|
13
|
+
* of committing a document, failing `goto` with "Download is starting" —
|
|
14
|
+
* even though the response itself arrived and is decodable. The last
|
|
15
|
+
* main-frame response is therefore tracked on every fetch, and such failed
|
|
16
|
+
* `goto`s are recovered — the status-only shape into the status-bearing
|
|
17
|
+
* result ({@link recoverUnrenderedStatus}), the download shape via a
|
|
18
|
+
* refetch through the page's API request context
|
|
19
|
+
* ({@link recoverDownloadResponse}) — instead of a `WEB_PROVIDER_ERROR`.
|
|
6
20
|
*
|
|
7
21
|
* Lifecycle: the local backend launches a browser per fetch and closes it —
|
|
8
22
|
* nothing outlives the call. The CDP backend keeps ONE shared connection to
|
|
@@ -46,7 +60,7 @@ import type { ResolvedConfig } from './config.ts'
|
|
|
46
60
|
import { CdpConnectionPool } from './cdp-pool.ts'
|
|
47
61
|
import { htmlToMarkdown } from './markdown.ts'
|
|
48
62
|
import { resolveCdpBackend, resolvePlaywrightBackend } from './playwright-resolve.ts'
|
|
49
|
-
import type { PlaywrightBrowser, PlaywrightContext, PlaywrightPage, PlaywrightResponse, PlaywrightRoute } from './types.ts'
|
|
63
|
+
import type { PlaywrightBrowser, PlaywrightContext, PlaywrightDownload, PlaywrightPage, PlaywrightResponse, PlaywrightRoute } from './types.ts'
|
|
50
64
|
|
|
51
65
|
/** Stable id this provider registers under (the bundle patch pins it). */
|
|
52
66
|
export const PLAYWRIGHT_FETCH_PROVIDER_ID = 'playwright'
|
|
@@ -364,6 +378,7 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
364
378
|
const lease = await this.cdpPool.acquire(endpoint, timeout, effectiveContextMode(config))
|
|
365
379
|
await installResourceFilter(lease.page)
|
|
366
380
|
guardPopups(lease.page)
|
|
381
|
+
guardDownloads(lease.page)
|
|
367
382
|
return {
|
|
368
383
|
browser: lease.browser,
|
|
369
384
|
context: lease.context,
|
|
@@ -387,6 +402,7 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
387
402
|
const page = await context.newPage()
|
|
388
403
|
await installResourceFilter(page)
|
|
389
404
|
guardPopups(page)
|
|
405
|
+
guardDownloads(page)
|
|
390
406
|
return { browser, context, page }
|
|
391
407
|
} catch (error: unknown) {
|
|
392
408
|
// A partial setup (browser launched, then newContext/newPage failed)
|
|
@@ -414,18 +430,23 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
414
430
|
): Promise<WebFetchResult> {
|
|
415
431
|
const page = session.page
|
|
416
432
|
const challengeWaitMs = effectiveChallengeWaitMs(config)
|
|
433
|
+
// The tracker is always on: the challenge wait needs the LAST main-frame
|
|
434
|
+
// response (feature switch below only gates the waiting), and status
|
|
435
|
+
// recovery needs whatever response arrived before a failed goto.
|
|
436
|
+
const tracker = trackMainFrameResponses(page)
|
|
417
437
|
// Feature switch: 0 keeps the exact legacy (pre-0.2.5) behavior — the
|
|
418
438
|
// first response decides, no waiting — an escape hatch and the A/B
|
|
419
439
|
// baseline every test proves the bug against.
|
|
420
|
-
const
|
|
421
|
-
|
|
422
|
-
|
|
440
|
+
const first = await this.navigate(page, url, tracker, deadline, config.denoise)
|
|
441
|
+
if (first.recovered !== undefined) return first.recovered
|
|
442
|
+
let response = first.response
|
|
443
|
+
tracker.seed(response)
|
|
423
444
|
|
|
424
445
|
let challengeEntryResponse: PlaywrightResponse | null = null
|
|
425
446
|
if (challengeWaitMs > 0) {
|
|
426
447
|
let attemptsLeft = effectiveChallengeRetries(config) + 1
|
|
427
448
|
for (;;) {
|
|
428
|
-
const verdict = await this.verdictAfterLoad(page, tracker
|
|
449
|
+
const verdict = await this.verdictAfterLoad(page, tracker.last() ?? response)
|
|
429
450
|
if (verdict === 'blocked') {
|
|
430
451
|
throw new WebError(
|
|
431
452
|
`the site hard-blocked this fetch at its Cloudflare edge (waiting cannot clear it): ${page.url()}`,
|
|
@@ -433,7 +454,7 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
433
454
|
)
|
|
434
455
|
}
|
|
435
456
|
if (verdict !== 'challenge') break
|
|
436
|
-
challengeEntryResponse = tracker
|
|
457
|
+
challengeEntryResponse = tracker.last() ?? response
|
|
437
458
|
const cleared = await this.waitForChallengeClear(page, deadline, challengeWaitMs)
|
|
438
459
|
if (cleared) {
|
|
439
460
|
// The settled document may be the next round of a CHAINED
|
|
@@ -446,7 +467,7 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
446
467
|
if (!chained) break
|
|
447
468
|
}
|
|
448
469
|
if (--attemptsLeft <= 0) {
|
|
449
|
-
const lastStatus = (tracker
|
|
470
|
+
const lastStatus = (tracker.last() ?? response)?.status()
|
|
450
471
|
throw new WebError(
|
|
451
472
|
`the site kept serving a Cloudflare challenge (last status ${lastStatus === undefined ? 'unknown' : String(lastStatus)}) for up to ${String(challengeWaitMs)}ms across ${String(effectiveChallengeRetries(config) + 1)} attempt(s); the browser did not clear it naturally — retry later, raise challengeWaitMs, or use a profile whose browser already holds clearance`,
|
|
452
473
|
WEB_FETCH_CHALLENGE_CODE,
|
|
@@ -455,14 +476,16 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
455
476
|
// Same page, same context — after an expired window OR a chained
|
|
456
477
|
// round: any clearance cookies already earned stay in the jar for
|
|
457
478
|
// this one retry, then everything is torn down as usual.
|
|
458
|
-
|
|
459
|
-
|
|
479
|
+
const retry = await this.navigate(page, url, tracker, deadline, config.denoise)
|
|
480
|
+
if (retry.recovered !== undefined) return retry.recovered
|
|
481
|
+
response = retry.response
|
|
482
|
+
tracker.seed(response)
|
|
460
483
|
}
|
|
461
484
|
}
|
|
462
485
|
|
|
463
486
|
// Final document: the LAST main-frame response when the challenge wait
|
|
464
487
|
// ran (the real page reloads in), else the response goto returned.
|
|
465
|
-
const finalResponse = (challengeWaitMs > 0 ? tracker
|
|
488
|
+
const finalResponse = (challengeWaitMs > 0 ? tracker.last() : undefined) ?? response
|
|
466
489
|
const kind = classifyContentType(finalResponse?.headers()['content-type'])
|
|
467
490
|
if (kind === undefined) {
|
|
468
491
|
throw new WebError(`unsupported content type "${finalResponse?.headers()['content-type'] ?? 'unknown'}"`, 'WEB_UNSUPPORTED_CONTENT_TYPE')
|
|
@@ -497,6 +520,42 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
497
520
|
: result
|
|
498
521
|
}
|
|
499
522
|
|
|
523
|
+
/**
|
|
524
|
+
* One tracked `goto`: the navigation response on success, or — when
|
|
525
|
+
* Chromium failed the navigation for a reason that is not a transport
|
|
526
|
+
* error — the recovered finished result. Two such reasons are recovered:
|
|
527
|
+
* the HTTP status it received (an error status with an empty body, or a
|
|
528
|
+
* status-only 204/205 — {@link recoverUnrenderedStatus}), and the response
|
|
529
|
+
* being a download the browser refuses to commit as a document
|
|
530
|
+
* (`Content-Disposition: attachment` — {@link recoverDownloadResponse}).
|
|
531
|
+
*
|
|
532
|
+
* @returns the response `goto` resolved with, plus `recovered` when the
|
|
533
|
+
* navigation failure was recovered into a finished result the caller must
|
|
534
|
+
* return verbatim (the throw never escapes in that case).
|
|
535
|
+
*/
|
|
536
|
+
private async navigate(
|
|
537
|
+
page: PlaywrightPage,
|
|
538
|
+
url: URL,
|
|
539
|
+
tracker: MainFrameTracker,
|
|
540
|
+
deadline: Deadline,
|
|
541
|
+
denoise: boolean,
|
|
542
|
+
): Promise<{ response: PlaywrightResponse | null; recovered?: WebFetchResult }> {
|
|
543
|
+
try {
|
|
544
|
+
const response = await page.goto(url.toString(), { waitUntil: 'domcontentloaded', timeout: deadline.remainingMs() })
|
|
545
|
+
return { response }
|
|
546
|
+
} catch (error) {
|
|
547
|
+
// Our own deadline/abort closing the page mid-navigation is a
|
|
548
|
+
// cancellation (translateError classifies it) — never a status fact.
|
|
549
|
+
if (!deadline.signal.aborted) {
|
|
550
|
+
const recovered = await recoverUnrenderedStatus(error, tracker, url, denoise)
|
|
551
|
+
if (recovered !== undefined) return { response: null, recovered }
|
|
552
|
+
const download = await recoverDownloadResponse(error, page, tracker, url, deadline, denoise)
|
|
553
|
+
if (download !== undefined) return { response: null, recovered: download }
|
|
554
|
+
}
|
|
555
|
+
throw error
|
|
556
|
+
}
|
|
557
|
+
}
|
|
558
|
+
|
|
500
559
|
/**
|
|
501
560
|
* Whether the freshly loaded document is a Cloudflare interstitial: the
|
|
502
561
|
* response-level signals first (the documented `cf-mitigated` header, then
|
|
@@ -608,6 +667,186 @@ function trackMainFrameResponses(page: PlaywrightPage): MainFrameTracker {
|
|
|
608
667
|
}
|
|
609
668
|
}
|
|
610
669
|
|
|
670
|
+
/**
|
|
671
|
+
* Whether a `goto` throw is Chromium refusing to RENDER a response it
|
|
672
|
+
* received — not a transport failure. Two net errors, both documented as
|
|
673
|
+
* rendering decisions rather than network ones:
|
|
674
|
+
*
|
|
675
|
+
* - `ERR_HTTP_RESPONSE_CODE_FAILURE` — an error status whose empty body
|
|
676
|
+
* leaves nothing to commit (httpbin's `/status/N`); net_error_list.h
|
|
677
|
+
* describes the code as "only used by certain APIs that interpret the
|
|
678
|
+
* HTTP response itself".
|
|
679
|
+
* - `ERR_ABORTED` on a status-only 204/205 — Chromium never renders those
|
|
680
|
+
* documents (`NavigationRequest` aborts them), while the response is
|
|
681
|
+
* final and the seam contract wants it as a result.
|
|
682
|
+
*
|
|
683
|
+
* @param error - the value `page.goto` rejected with.
|
|
684
|
+
* @param status - the status of the last tracked main-frame response.
|
|
685
|
+
* @returns true when the navigation failed for the status it received.
|
|
686
|
+
*/
|
|
687
|
+
function isUnrenderedStatusFailure(error: unknown, status: number): boolean {
|
|
688
|
+
if (!(error instanceof Error)) return false
|
|
689
|
+
const message = error.message
|
|
690
|
+
if (message.includes('ERR_HTTP_RESPONSE_CODE_FAILURE')) return true
|
|
691
|
+
return message.includes('ERR_ABORTED') && (status === 204 || status === 205)
|
|
692
|
+
}
|
|
693
|
+
|
|
694
|
+
/**
|
|
695
|
+
* Recover the response behind a status-only navigation failure into the
|
|
696
|
+
* result the web seam's contract demands — "a successful network fetch of a
|
|
697
|
+
* non-2xx response is a result, not an error: the status code is part of the
|
|
698
|
+
* fetched resource state" (`@deepseek-ai/dsh-web` types). The main-frame
|
|
699
|
+
* response arrives (and the tracker records it) before Chromium fails the
|
|
700
|
+
* navigation, so its status, headers, and URL are recoverable; only the body
|
|
701
|
+
* dies with the unrendered loader, and this failure class only fires on
|
|
702
|
+
* empty bodies — so an unreadable body decodes as the empty string it is.
|
|
703
|
+
*
|
|
704
|
+
* Deliberately NOT recovered: genuine transport errors (they carry no
|
|
705
|
+
* response), and our own deadline/abort (the caller classifies those).
|
|
706
|
+
*
|
|
707
|
+
* @param error - the value `page.goto` rejected with.
|
|
708
|
+
* @param tracker - the fetch's main-frame response record.
|
|
709
|
+
* @param url - the request URL (fallback when the response carries none).
|
|
710
|
+
* @param denoise - the settings toggle, honored as on the normal HTML path.
|
|
711
|
+
* @returns the finished fetch result, or undefined when the failure is not
|
|
712
|
+
* a recoverable status-only navigation (the throw should propagate).
|
|
713
|
+
*/
|
|
714
|
+
async function recoverUnrenderedStatus(
|
|
715
|
+
error: unknown,
|
|
716
|
+
tracker: MainFrameTracker,
|
|
717
|
+
url: URL,
|
|
718
|
+
denoise: boolean,
|
|
719
|
+
): Promise<WebFetchResult | undefined> {
|
|
720
|
+
const response = tracker.last()
|
|
721
|
+
if (response === null) return undefined
|
|
722
|
+
const status = response.status()
|
|
723
|
+
if (!isUnrenderedStatusFailure(error, status)) return undefined
|
|
724
|
+
const headers = response.headers()
|
|
725
|
+
// A challenge edge answering with nothing to render cannot be waited out
|
|
726
|
+
// (no document ever loaded to clear); keep the plugin's invariant of
|
|
727
|
+
// never passing challenge responses through as content.
|
|
728
|
+
if (classifyChallengeResponse(status, headers) === 'challenge') {
|
|
729
|
+
throw new WebError(
|
|
730
|
+
`the site answered with a Cloudflare challenge response (HTTP ${String(status)}) whose empty body the browser refused to render; waiting cannot clear it — retry later or use a profile whose browser already holds clearance`,
|
|
731
|
+
WEB_FETCH_CHALLENGE_CODE,
|
|
732
|
+
)
|
|
733
|
+
}
|
|
734
|
+
const kind = classifyContentType(headers['content-type'])
|
|
735
|
+
if (kind === undefined) {
|
|
736
|
+
throw new WebError(`unsupported content type "${headers['content-type'] ?? 'unknown'}"`, 'WEB_UNSUPPORTED_CONTENT_TYPE')
|
|
737
|
+
}
|
|
738
|
+
const finalUrl = response.url?.() ?? url.toString()
|
|
739
|
+
// The unrenderable loader may refuse the body read too; this failure
|
|
740
|
+
// class only fires on empty bodies, so '' is the honest fallback.
|
|
741
|
+
let content = ''
|
|
742
|
+
try { content = await response.text() } catch { content = '' }
|
|
743
|
+
if (kind === 'text' || content === '') {
|
|
744
|
+
// Empty HTML needs no pipeline — the empty string IS the body.
|
|
745
|
+
return capResult(finalUrl, status, { kind: 'text', content })
|
|
746
|
+
}
|
|
747
|
+
if (!denoise) return capResult(finalUrl, status, { kind: 'html', content })
|
|
748
|
+
const { markdown } = htmlToMarkdown(content, finalUrl)
|
|
749
|
+
return capResult(finalUrl, status, { kind: 'text', content: markdown })
|
|
750
|
+
}
|
|
751
|
+
|
|
752
|
+
/**
|
|
753
|
+
* Whether a `goto` throw is Chromium handing the navigation to its download
|
|
754
|
+
* manager instead of committing a document. Playwright rejects with
|
|
755
|
+
* "Download is starting" when the navigation response carries
|
|
756
|
+
* `Content-Disposition: attachment` (publish.twitter.com's oembed endpoint —
|
|
757
|
+
* the workaround URL agents use to read a tweet — serves exactly that:
|
|
758
|
+
* decodable `application/json` the browser still refuses to render). The
|
|
759
|
+
* attachment disposition on the tracked response is a second,
|
|
760
|
+
* engine-version-proof signal for engines that report a plain `ERR_ABORTED`
|
|
761
|
+
* instead; note `ERR_ABORTED` alone is NOT enough (the status recovery
|
|
762
|
+
* above legitimately owns the 204/205 shape, and SPA cancels abort too).
|
|
763
|
+
*
|
|
764
|
+
* @param error - the value `page.goto` rejected with.
|
|
765
|
+
* @param response - the last tracked main-frame response, if any.
|
|
766
|
+
* @returns true when the navigation failed because it became a download.
|
|
767
|
+
*/
|
|
768
|
+
function isDownloadFailure(error: unknown, response: PlaywrightResponse | null): boolean {
|
|
769
|
+
if (!(error instanceof Error)) return false
|
|
770
|
+
if (error.message.includes('Download is starting')) return true
|
|
771
|
+
if (!error.message.includes('ERR_ABORTED')) return false
|
|
772
|
+
const disposition = (response?.headers()['content-disposition'] ?? '').toLowerCase()
|
|
773
|
+
return disposition.includes('attachment')
|
|
774
|
+
}
|
|
775
|
+
|
|
776
|
+
/**
|
|
777
|
+
* Recover a download-turned navigation into the finished fetch result the
|
|
778
|
+
* seam's contract demands. The resource itself is often perfectly decodable
|
|
779
|
+
* — the oembed endpoint's `application/json` classifies as `text` — the
|
|
780
|
+
* browser just will not commit it as a document, and the response it tracked
|
|
781
|
+
* cannot yield its body either (Chromium streams a download to its download
|
|
782
|
+
* manager, so `Network.getResponseBody` answers "No resource with given
|
|
783
|
+
* identifier found"; both verified against a real browser). The body is
|
|
784
|
+
* therefore refetched through the page's API request context: plain HTTP
|
|
785
|
+
* from the Playwright client — same cookie jar, same URL, no render-or-
|
|
786
|
+
* download decision — which works identically on the local and CDP
|
|
787
|
+
* backends because it never touches the remote artifact.
|
|
788
|
+
*
|
|
789
|
+
* Undecodable downloads (a PDF, a ZIP) still fail with
|
|
790
|
+
* `WEB_UNSUPPORTED_CONTENT_TYPE`, classified from what the navigation
|
|
791
|
+
* actually served; a playwright build without a request context surfaces a
|
|
792
|
+
* clear "the browser will not render this" error instead of the raw
|
|
793
|
+
* "Download is starting".
|
|
794
|
+
*
|
|
795
|
+
* @param error - the value `page.goto` rejected with.
|
|
796
|
+
* @param page - the fetch's tab, for its API request context.
|
|
797
|
+
* @param tracker - the fetch's main-frame response record.
|
|
798
|
+
* @param url - the request URL (fallback when the response carries none).
|
|
799
|
+
* @param deadline - the fetch budget, applied to the refetch.
|
|
800
|
+
* @param denoise - the settings toggle, honored as on the normal HTML path.
|
|
801
|
+
* @returns the finished fetch result, or undefined when the failure is not
|
|
802
|
+
* a download navigation (the throw should propagate).
|
|
803
|
+
*/
|
|
804
|
+
async function recoverDownloadResponse(
|
|
805
|
+
error: unknown,
|
|
806
|
+
page: PlaywrightPage,
|
|
807
|
+
tracker: MainFrameTracker,
|
|
808
|
+
url: URL,
|
|
809
|
+
deadline: Deadline,
|
|
810
|
+
denoise: boolean,
|
|
811
|
+
): Promise<WebFetchResult | undefined> {
|
|
812
|
+
const tracked = tracker.last()
|
|
813
|
+
if (!isDownloadFailure(error, tracked)) return undefined
|
|
814
|
+
if (tracked !== null) {
|
|
815
|
+
// What the site served the navigation is the truth about the resource:
|
|
816
|
+
// an undecodable download (PDF/ZIP/…) has no business being refetched.
|
|
817
|
+
const trackedKind = classifyContentType(tracked.headers()['content-type'])
|
|
818
|
+
if (trackedKind === undefined) {
|
|
819
|
+
throw new WebError(`unsupported content type "${tracked.headers()['content-type'] ?? 'unknown'}" (served as a download)`, 'WEB_UNSUPPORTED_CONTENT_TYPE')
|
|
820
|
+
}
|
|
821
|
+
}
|
|
822
|
+
if (page.request === undefined) {
|
|
823
|
+
throw new WebError(
|
|
824
|
+
'the response is a file download (Content-Disposition: attachment) the browser will not render, and this playwright build exposes no request context to refetch it',
|
|
825
|
+
'WEB_PROVIDER_ERROR',
|
|
826
|
+
{ cause: error },
|
|
827
|
+
)
|
|
828
|
+
}
|
|
829
|
+
const fetchUrl = tracked?.url?.() ?? url.toString()
|
|
830
|
+
const refetched = await page.request.get(fetchUrl, { timeout: deadline.remainingMs(), signal: deadline.signal })
|
|
831
|
+
const headers = refetched.headers()
|
|
832
|
+
// The refetch speaks for the bytes it returned; only a content-type-free
|
|
833
|
+
// answer falls back to what the navigation served.
|
|
834
|
+
const servedType = (headers['content-type'] ?? '').trim() !== '' ? headers['content-type'] : tracked?.headers()['content-type']
|
|
835
|
+
const kind = classifyContentType(servedType)
|
|
836
|
+
if (kind === undefined) {
|
|
837
|
+
throw new WebError(`unsupported content type "${servedType ?? 'unknown'}"`, 'WEB_UNSUPPORTED_CONTENT_TYPE')
|
|
838
|
+
}
|
|
839
|
+
const finalUrl = refetched.url?.() ?? fetchUrl
|
|
840
|
+
const statusCode = refetched.status()
|
|
841
|
+
const content = await refetched.text()
|
|
842
|
+
if (kind === 'text') {
|
|
843
|
+
return capResult(finalUrl, statusCode, { kind: 'text', content })
|
|
844
|
+
}
|
|
845
|
+
if (!denoise) return capResult(finalUrl, statusCode, { kind: 'html', content })
|
|
846
|
+
const { markdown } = htmlToMarkdown(content, finalUrl)
|
|
847
|
+
return capResult(finalUrl, statusCode, { kind: 'text', content: markdown })
|
|
848
|
+
}
|
|
849
|
+
|
|
611
850
|
/**
|
|
612
851
|
* Probe the live document for challenge markers. Prefers the in-page probe
|
|
613
852
|
* (`page.evaluate`) — the only thing that can see SPA-style clears — and
|
|
@@ -700,6 +939,23 @@ async function installResourceFilter(owner: {
|
|
|
700
939
|
}
|
|
701
940
|
}
|
|
702
941
|
|
|
942
|
+
/**
|
|
943
|
+
* Cancel any download Chromium starts on the fetch's tab. A navigation whose
|
|
944
|
+
* response carries `Content-Disposition: attachment` never commits a
|
|
945
|
+
* document — `goto` rejects with "Download is starting" and the recovery
|
|
946
|
+
* refetches the bytes (see {@link recoverDownloadResponse}) — but the
|
|
947
|
+
* browser still hands the original body to its download manager, which in
|
|
948
|
+
* profile mode would drop files into the remote machine's download dir.
|
|
949
|
+
* Best-effort: a page without the listener just loses the guard.
|
|
950
|
+
*/
|
|
951
|
+
function guardDownloads(page: PlaywrightPage): void {
|
|
952
|
+
try {
|
|
953
|
+
page.on?.('download', (download: PlaywrightDownload) => { void download.cancel().catch(() => {}) })
|
|
954
|
+
} catch {
|
|
955
|
+
// keep going without the guard
|
|
956
|
+
}
|
|
957
|
+
}
|
|
958
|
+
|
|
703
959
|
/**
|
|
704
960
|
* Close any popup a fetched page spawns so nothing outlives the fetch's tab
|
|
705
961
|
* — `page.close()` does not auto-close windows the page opened, and in
|
package/src/types.ts
CHANGED
|
@@ -22,6 +22,26 @@ export interface PlaywrightResponse {
|
|
|
22
22
|
request?(): PlaywrightRequest
|
|
23
23
|
}
|
|
24
24
|
|
|
25
|
+
/**
|
|
26
|
+
* A download Chromium started instead of committing a document (a response
|
|
27
|
+
* with `Content-Disposition: attachment`).
|
|
28
|
+
*/
|
|
29
|
+
export interface PlaywrightDownload {
|
|
30
|
+
cancel(): Promise<void>
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* The page's `APIRequestContext` surface the download recovery refetches
|
|
35
|
+
* through — plain HTTP from the Playwright client, the context's cookie jar,
|
|
36
|
+
* and none of Chromium's render-or-download semantics. Reuses
|
|
37
|
+
* {@link PlaywrightResponse} because that is structurally what an
|
|
38
|
+
* `APIResponse` is for the members this plugin reads. Absent on fakes that
|
|
39
|
+
* do not exercise the recovery.
|
|
40
|
+
*/
|
|
41
|
+
export interface PlaywrightRequestContext {
|
|
42
|
+
get(url: string, options?: { timeout?: number; signal?: AbortSignal }): Promise<PlaywrightResponse>
|
|
43
|
+
}
|
|
44
|
+
|
|
25
45
|
/** The request side of a response, for main-frame filtering. */
|
|
26
46
|
export interface PlaywrightRequest {
|
|
27
47
|
/** True for navigations (document loads and their redirect hops). */
|
|
@@ -52,6 +72,13 @@ export interface PlaywrightPage {
|
|
|
52
72
|
* document). Absent on minimal fakes (content polling covers them).
|
|
53
73
|
*/
|
|
54
74
|
on?(event: 'response', listener: (response: PlaywrightResponse) => void): unknown
|
|
75
|
+
/**
|
|
76
|
+
* Download notification — a navigation the browser handed to its download
|
|
77
|
+
* manager instead of committing a document (`Content-Disposition:
|
|
78
|
+
* attachment`); the fetch's guard cancels it so nothing lands on disk.
|
|
79
|
+
* Absent on minimal fakes (the recovery does not depend on it).
|
|
80
|
+
*/
|
|
81
|
+
on?(event: 'download', listener: (download: PlaywrightDownload) => void): unknown
|
|
55
82
|
/**
|
|
56
83
|
* Evaluate an expression in the page — the challenge probe's live-DOM
|
|
57
84
|
* path. Absent on minimal fakes (content polling covers them).
|
|
@@ -59,6 +86,11 @@ export interface PlaywrightPage {
|
|
|
59
86
|
evaluate?(script: string, arg?: unknown): Promise<unknown>
|
|
60
87
|
/** The main frame handle; compare with `request.frame()` for filtering. */
|
|
61
88
|
mainFrame?(): unknown
|
|
89
|
+
/**
|
|
90
|
+
* The page's API request context — the download recovery's refetch path
|
|
91
|
+
* (see {@link PlaywrightRequestContext}). Absent on minimal fakes.
|
|
92
|
+
*/
|
|
93
|
+
request?: PlaywrightRequestContext
|
|
62
94
|
}
|
|
63
95
|
|
|
64
96
|
/**
|