dsh-web-fetch-playwright 0.2.10 → 0.2.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -34,7 +34,7 @@ web_fetch (tool-web)
34
34
  └─ Markdown (or raw HTML when denoise is off)
35
35
  ```
36
36
 
37
- **Non-2xx statuses are results, not errors** — the web seam's contract (`@deepseek-ai/dsh-web`): a fetched 404/418/500 comes back with its status code, exactly what `dsh-web-fetch-http` does. Chromium complicates this for *status-only* responses — an error status with an empty body (httpbin's `/status/404`) fails the whole navigation with `net::ERR_HTTP_RESPONSE_CODE_FAILURE`, and a 204/205 is aborted (`net::ERR_ABORTED`) because nothing is renderable. The provider therefore tracks the last main-frame response on every fetch (that response arrives before Chromium fails the navigation) and recovers such failures into the status-bearing result, so `web_fetch` reports `Fetched … (HTTP 404)` instead of a `WEB_PROVIDER_ERROR` throw. A recovered response that is itself a Cloudflare challenge edge still fails with `WEB_FETCH_CHALLENGE`; transport errors and cancellations stay errors.
37
+ **Non-2xx statuses are results, not errors** — the web seam's contract (`@deepseek-ai/dsh-web`): a fetched 404/418/500 comes back with its status code, exactly what `dsh-web-fetch-http` does. Chromium complicates this for *status-only* responses — an error status with an empty body (httpbin's `/status/404`) fails the whole navigation with `net::ERR_HTTP_RESPONSE_CODE_FAILURE`, and a 204/205 is aborted (`net::ERR_ABORTED`) because nothing is renderable. The provider therefore tracks the last main-frame response on every fetch (that response arrives before Chromium fails the navigation) and recovers such failures into the status-bearing result, so `web_fetch` reports `Fetched … (HTTP 404)` instead of a `WEB_PROVIDER_ERROR` throw. The same recovery covers *download* responses — a body served with `Content-Disposition: attachment` (publish.twitter.com's oembed endpoint, the URL agents use to read a tweet, serves its JSON exactly that way) never commits a document: `goto` rejects with `Download is starting` and the body is gone from the network stack, so the provider refetches it through the page's API request context and returns the decodable content; undecodable downloads (a PDF, a ZIP) still fail with `WEB_UNSUPPORTED_CONTENT_TYPE`. A recovered response that is itself a Cloudflare challenge edge still fails with `WEB_FETCH_CHALLENGE`; transport errors and cancellations stay errors.
38
38
 
39
39
  ## Requirements
40
40
 
package/README.zh-CN.md CHANGED
@@ -34,7 +34,7 @@ web_fetch (tool-web)
34
34
  └─ Markdown(关闭降噪时返回原始 HTML)
35
35
  ```
36
36
 
37
- **非 2xx 状态码是结果而非错误** —— 这是 web seam 的契约(`@deepseek-ai/dsh-web`):抓取到的 404/418/500 会连同状态码一起正常返回,与 `dsh-web-fetch-http` 的行为一致。但对*仅有状态*的响应 Chromium 会把事情搞砸 —— 空 body 的错误状态(httpbin 的 `/status/404`)会让整个导航以 `net::ERR_HTTP_RESPONSE_CODE_FAILURE` 失败,204/205 则因无可渲染内容被中止(`net::ERR_ABORTED`)。因此 provider 在每次抓取中都跟踪最后一个主框架响应(该响应先于 Chromium 判定导航失败到达),并把这类失败恢复成带状态码的正常结果,`web_fetch` 会报告 `Fetched … (HTTP 404)` 而不是抛出 `WEB_PROVIDER_ERROR`。若恢复出的响应本身来自 Cloudflare 挑战边缘,仍以 `WEB_FETCH_CHALLENGE` 失败;传输错误与取消也依旧按错误处理。
37
+ **非 2xx 状态码是结果而非错误** —— 这是 web seam 的契约(`@deepseek-ai/dsh-web`):抓取到的 404/418/500 会连同状态码一起正常返回,与 `dsh-web-fetch-http` 的行为一致。但对*仅有状态*的响应 Chromium 会把事情搞砸 —— 空 body 的错误状态(httpbin 的 `/status/404`)会让整个导航以 `net::ERR_HTTP_RESPONSE_CODE_FAILURE` 失败,204/205 则因无可渲染内容被中止(`net::ERR_ABORTED`)。因此 provider 在每次抓取中都跟踪最后一个主框架响应(该响应先于 Chromium 判定导航失败到达),并把这类失败恢复成带状态码的正常结果,`web_fetch` 会报告 `Fetched … (HTTP 404)` 而不是抛出 `WEB_PROVIDER_ERROR`。同样的恢复也覆盖*下载型*响应 —— 带 `Content-Disposition: attachment` 的 body(publish.twitter.com 的 oembed 端点——agent 读取推文常用的替代 URL——返回的 JSON 正是如此)永远不会被提交为文档:`goto` 以 `Download is starting` 拒绝,且 body 已从网络栈中流失,provider 因此通过页面的 API request context 重新拉取并返回可解码内容;不可解码的下载(PDF、ZIP)仍以 `WEB_UNSUPPORTED_CONTENT_TYPE` 失败。若恢复出的响应本身来自 Cloudflare 挑战边缘,仍以 `WEB_FETCH_CHALLENGE` 失败;传输错误与取消也依旧按错误处理。
38
38
 
39
39
  ## 环境要求
40
40
 
package/lib/client.js CHANGED
@@ -413,11 +413,11 @@ window.__ModuleLoader__.load({
413
413
  }
414
414
  var PluginCard_module_css_default = {
415
415
  "readOnly": "UM8-Rq_readOnly",
416
- "form": "UM8-Rq_form",
417
416
  "failed": "UM8-Rq_failed",
418
- "footer": "UM8-Rq_footer",
417
+ "discard": "UM8-Rq_discard",
419
418
  "save": "UM8-Rq_save",
420
- "discard": "UM8-Rq_discard"
419
+ "footer": "UM8-Rq_footer",
420
+ "form": "UM8-Rq_form"
421
421
  };
422
422
  //#endregion
423
423
  //#region src/client/PluginCard.tsx
@@ -477,26 +477,26 @@ window.__ModuleLoader__.load({
477
477
  document.head.appendChild(tag);
478
478
  }
479
479
  var fields_module_css_default = {
480
+ "radioLabel": "CFSaVG_radioLabel",
481
+ "radioHint": "CFSaVG_radioHint",
480
482
  "input": "CFSaVG_input",
481
- "inputInvalid": "CFSaVG_inputInvalid",
482
- "hintIndented": "CFSaVG_hintIndented",
483
483
  "radioText": "CFSaVG_radioText",
484
- "radioPick": "CFSaVG_radioPick",
485
- "radioLabel": "CFSaVG_radioLabel",
486
484
  "invalid": "CFSaVG_invalid",
487
- "radioHint": "CFSaVG_radioHint",
488
- "radioOption": "CFSaVG_radioOption",
489
- "badges": "CFSaVG_badges",
485
+ "radioGroup": "CFSaVG_radioGroup",
486
+ "radioContent": "CFSaVG_radioContent",
490
487
  "label": "CFSaVG_label",
491
- "hint": "CFSaVG_hint",
492
- "badge": "CFSaVG_badge",
493
488
  "field": "CFSaVG_field",
494
- "reset": "CFSaVG_reset",
489
+ "hint": "CFSaVG_hint",
495
490
  "fieldEmbedded": "CFSaVG_fieldEmbedded",
496
- "radioGroup": "CFSaVG_radioGroup",
497
- "checkboxRow": "CFSaVG_checkboxRow",
491
+ "badges": "CFSaVG_badges",
492
+ "reset": "CFSaVG_reset",
498
493
  "head": "CFSaVG_head",
499
- "radioContent": "CFSaVG_radioContent"
494
+ "radioOption": "CFSaVG_radioOption",
495
+ "inputInvalid": "CFSaVG_inputInvalid",
496
+ "hintIndented": "CFSaVG_hintIndented",
497
+ "badge": "CFSaVG_badge",
498
+ "radioPick": "CFSaVG_radioPick",
499
+ "checkboxRow": "CFSaVG_checkboxRow"
500
500
  };
501
501
  //#endregion
502
502
  //#region src/client/fields.tsx
package/lib/index.js CHANGED
@@ -998,11 +998,16 @@ function existsAsFile(path) {
998
998
  * an error. Chromium can make that awkward: it refuses to render status-only
999
999
  * responses (error statuses with an empty body — httpbin's `/status/N` —
1000
1000
  * plus 204/205), failing the whole navigation with
1001
- * `net::ERR_HTTP_RESPONSE_CODE_FAILURE` / `net::ERR_ABORTED` even though the
1002
- * response itself arrived. The last main-frame response is therefore tracked
1003
- * on every fetch, and such a failed `goto` is recovered into the
1004
- * status-bearing result the contract demands instead of a
1005
- * `WEB_PROVIDER_ERROR` (see {@link recoverUnrenderedStatus}).
1001
+ * `net::ERR_HTTP_RESPONSE_CODE_FAILURE` / `net::ERR_ABORTED`, and it hands a
1002
+ * `Content-Disposition: attachment` response (publish.twitter.com's oembed
1003
+ * JSON, the URL agents use to read a tweet) to its download manager instead
1004
+ * of committing a document, failing `goto` with "Download is starting" —
1005
+ * even though the response itself arrived and is decodable. The last
1006
+ * main-frame response is therefore tracked on every fetch, and such failed
1007
+ * `goto`s are recovered — the status-only shape into the status-bearing
1008
+ * result ({@link recoverUnrenderedStatus}), the download shape via a
1009
+ * refetch through the page's API request context
1010
+ * ({@link recoverDownloadResponse}) — instead of a `WEB_PROVIDER_ERROR`.
1006
1011
  *
1007
1012
  * Lifecycle: the local backend launches a browser per fetch and closes it —
1008
1013
  * nothing outlives the call. The CDP backend keeps ONE shared connection to
@@ -1284,6 +1289,7 @@ var PlaywrightFetchProvider = class {
1284
1289
  const lease = await this.cdpPool.acquire(endpoint, timeout, effectiveContextMode(config));
1285
1290
  await installResourceFilter(lease.page);
1286
1291
  guardPopups(lease.page);
1292
+ guardDownloads(lease.page);
1287
1293
  return {
1288
1294
  browser: lease.browser,
1289
1295
  context: lease.context,
@@ -1307,6 +1313,7 @@ var PlaywrightFetchProvider = class {
1307
1313
  const page = await context.newPage();
1308
1314
  await installResourceFilter(page);
1309
1315
  guardPopups(page);
1316
+ guardDownloads(page);
1310
1317
  return {
1311
1318
  browser,
1312
1319
  context,
@@ -1385,9 +1392,12 @@ var PlaywrightFetchProvider = class {
1385
1392
  }
1386
1393
  /**
1387
1394
  * One tracked `goto`: the navigation response on success, or — when
1388
- * Chromium failed the navigation purely because of the HTTP status it
1389
- * received (an error status with an empty body, or a status-only 204/205)
1390
- * — the recovered status-bearing result from {@link recoverUnrenderedStatus}.
1395
+ * Chromium failed the navigation for a reason that is not a transport
1396
+ * error — the recovered finished result. Two such reasons are recovered:
1397
+ * the HTTP status it received (an error status with an empty body, or a
1398
+ * status-only 204/205 — {@link recoverUnrenderedStatus}), and the response
1399
+ * being a download the browser refuses to commit as a document
1400
+ * (`Content-Disposition: attachment` — {@link recoverDownloadResponse}).
1391
1401
  *
1392
1402
  * @returns the response `goto` resolved with, plus `recovered` when the
1393
1403
  * navigation failure was recovered into a finished result the caller must
@@ -1406,6 +1416,11 @@ var PlaywrightFetchProvider = class {
1406
1416
  response: null,
1407
1417
  recovered
1408
1418
  };
1419
+ const download = await recoverDownloadResponse(error, page, tracker, url, deadline, denoise);
1420
+ if (download !== void 0) return {
1421
+ response: null,
1422
+ recovered: download
1423
+ };
1409
1424
  }
1410
1425
  throw error;
1411
1426
  }
@@ -1570,6 +1585,89 @@ async function recoverUnrenderedStatus(error, tracker, url, denoise) {
1570
1585
  });
1571
1586
  }
1572
1587
  /**
1588
+ * Whether a `goto` throw is Chromium handing the navigation to its download
1589
+ * manager instead of committing a document. Playwright rejects with
1590
+ * "Download is starting" when the navigation response carries
1591
+ * `Content-Disposition: attachment` (publish.twitter.com's oembed endpoint —
1592
+ * the workaround URL agents use to read a tweet — serves exactly that:
1593
+ * decodable `application/json` the browser still refuses to render). The
1594
+ * attachment disposition on the tracked response is a second,
1595
+ * engine-version-proof signal for engines that report a plain `ERR_ABORTED`
1596
+ * instead; note `ERR_ABORTED` alone is NOT enough (the status recovery
1597
+ * above legitimately owns the 204/205 shape, and SPA cancels abort too).
1598
+ *
1599
+ * @param error - the value `page.goto` rejected with.
1600
+ * @param response - the last tracked main-frame response, if any.
1601
+ * @returns true when the navigation failed because it became a download.
1602
+ */
1603
+ function isDownloadFailure(error, response) {
1604
+ if (!(error instanceof Error)) return false;
1605
+ if (error.message.includes("Download is starting")) return true;
1606
+ if (!error.message.includes("ERR_ABORTED")) return false;
1607
+ return (response?.headers()["content-disposition"] ?? "").toLowerCase().includes("attachment");
1608
+ }
1609
+ /**
1610
+ * Recover a download-turned navigation into the finished fetch result the
1611
+ * seam's contract demands. The resource itself is often perfectly decodable
1612
+ * — the oembed endpoint's `application/json` classifies as `text` — the
1613
+ * browser just will not commit it as a document, and the response it tracked
1614
+ * cannot yield its body either (Chromium streams a download to its download
1615
+ * manager, so `Network.getResponseBody` answers "No resource with given
1616
+ * identifier found"; both verified against a real browser). The body is
1617
+ * therefore refetched through the page's API request context: plain HTTP
1618
+ * from the Playwright client — same cookie jar, same URL, no render-or-
1619
+ * download decision — which works identically on the local and CDP
1620
+ * backends because it never touches the remote artifact.
1621
+ *
1622
+ * Undecodable downloads (a PDF, a ZIP) still fail with
1623
+ * `WEB_UNSUPPORTED_CONTENT_TYPE`, classified from what the navigation
1624
+ * actually served; a playwright build without a request context surfaces a
1625
+ * clear "the browser will not render this" error instead of the raw
1626
+ * "Download is starting".
1627
+ *
1628
+ * @param error - the value `page.goto` rejected with.
1629
+ * @param page - the fetch's tab, for its API request context.
1630
+ * @param tracker - the fetch's main-frame response record.
1631
+ * @param url - the request URL (fallback when the response carries none).
1632
+ * @param deadline - the fetch budget, applied to the refetch.
1633
+ * @param denoise - the settings toggle, honored as on the normal HTML path.
1634
+ * @returns the finished fetch result, or undefined when the failure is not
1635
+ * a download navigation (the throw should propagate).
1636
+ */
1637
+ async function recoverDownloadResponse(error, page, tracker, url, deadline, denoise) {
1638
+ const tracked = tracker.last();
1639
+ if (!isDownloadFailure(error, tracked)) return void 0;
1640
+ if (tracked !== null) {
1641
+ if (classifyContentType(tracked.headers()["content-type"]) === void 0) throw new WebError(`unsupported content type "${tracked.headers()["content-type"] ?? "unknown"}" (served as a download)`, "WEB_UNSUPPORTED_CONTENT_TYPE");
1642
+ }
1643
+ if (page.request === void 0) throw new WebError("the response is a file download (Content-Disposition: attachment) the browser will not render, and this playwright build exposes no request context to refetch it", "WEB_PROVIDER_ERROR", { cause: error });
1644
+ const fetchUrl = tracked?.url?.() ?? url.toString();
1645
+ const refetched = await page.request.get(fetchUrl, {
1646
+ timeout: deadline.remainingMs(),
1647
+ signal: deadline.signal
1648
+ });
1649
+ const headers = refetched.headers();
1650
+ const servedType = (headers["content-type"] ?? "").trim() !== "" ? headers["content-type"] : tracked?.headers()["content-type"];
1651
+ const kind = classifyContentType(servedType);
1652
+ if (kind === void 0) throw new WebError(`unsupported content type "${servedType ?? "unknown"}"`, "WEB_UNSUPPORTED_CONTENT_TYPE");
1653
+ const finalUrl = refetched.url?.() ?? fetchUrl;
1654
+ const statusCode = refetched.status();
1655
+ const content = await refetched.text();
1656
+ if (kind === "text") return capResult(finalUrl, statusCode, {
1657
+ kind: "text",
1658
+ content
1659
+ });
1660
+ if (!denoise) return capResult(finalUrl, statusCode, {
1661
+ kind: "html",
1662
+ content
1663
+ });
1664
+ const { markdown } = htmlToMarkdown(content, finalUrl);
1665
+ return capResult(finalUrl, statusCode, {
1666
+ kind: "text",
1667
+ content: markdown
1668
+ });
1669
+ }
1670
+ /**
1573
1671
  * Probe the live document for challenge markers. Prefers the in-page probe
1574
1672
  * (`page.evaluate`) — the only thing that can see SPA-style clears — and
1575
1673
  * falls back to reading and classifying the serialized HTML when the backend
@@ -1654,6 +1752,22 @@ async function installResourceFilter(owner) {
1654
1752
  } catch {}
1655
1753
  }
1656
1754
  /**
1755
+ * Cancel any download Chromium starts on the fetch's tab. A navigation whose
1756
+ * response carries `Content-Disposition: attachment` never commits a
1757
+ * document — `goto` rejects with "Download is starting" and the recovery
1758
+ * refetches the bytes (see {@link recoverDownloadResponse}) — but the
1759
+ * browser still hands the original body to its download manager, which in
1760
+ * profile mode would drop files into the remote machine's download dir.
1761
+ * Best-effort: a page without the listener just loses the guard.
1762
+ */
1763
+ function guardDownloads(page) {
1764
+ try {
1765
+ page.on?.("download", (download) => {
1766
+ download.cancel().catch(() => {});
1767
+ });
1768
+ } catch {}
1769
+ }
1770
+ /**
1657
1771
  * Close any popup a fetched page spawns so nothing outlives the fetch's tab
1658
1772
  * — `page.close()` does not auto-close windows the page opened, and in
1659
1773
  * profile mode a stray tab would stay in the user's remote browser.
@@ -7,11 +7,16 @@
7
7
  * an error. Chromium can make that awkward: it refuses to render status-only
8
8
  * responses (error statuses with an empty body — httpbin's `/status/N` —
9
9
  * plus 204/205), failing the whole navigation with
10
- * `net::ERR_HTTP_RESPONSE_CODE_FAILURE` / `net::ERR_ABORTED` even though the
11
- * response itself arrived. The last main-frame response is therefore tracked
12
- * on every fetch, and such a failed `goto` is recovered into the
13
- * status-bearing result the contract demands instead of a
14
- * `WEB_PROVIDER_ERROR` (see {@link recoverUnrenderedStatus}).
10
+ * `net::ERR_HTTP_RESPONSE_CODE_FAILURE` / `net::ERR_ABORTED`, and it hands a
11
+ * `Content-Disposition: attachment` response (publish.twitter.com's oembed
12
+ * JSON, the URL agents use to read a tweet) to its download manager instead
13
+ * of committing a document, failing `goto` with "Download is starting" —
14
+ * even though the response itself arrived and is decodable. The last
15
+ * main-frame response is therefore tracked on every fetch, and such failed
16
+ * `goto`s are recovered — the status-only shape into the status-bearing
17
+ * result ({@link recoverUnrenderedStatus}), the download shape via a
18
+ * refetch through the page's API request context
19
+ * ({@link recoverDownloadResponse}) — instead of a `WEB_PROVIDER_ERROR`.
15
20
  *
16
21
  * Lifecycle: the local backend launches a browser per fetch and closes it —
17
22
  * nothing outlives the call. The CDP backend keeps ONE shared connection to
@@ -132,9 +137,12 @@ export declare class PlaywrightFetchProvider implements WebFetchProvider {
132
137
  private retrieve;
133
138
  /**
134
139
  * One tracked `goto`: the navigation response on success, or — when
135
- * Chromium failed the navigation purely because of the HTTP status it
136
- * received (an error status with an empty body, or a status-only 204/205)
137
- * — the recovered status-bearing result from {@link recoverUnrenderedStatus}.
140
+ * Chromium failed the navigation for a reason that is not a transport
141
+ * error — the recovered finished result. Two such reasons are recovered:
142
+ * the HTTP status it received (an error status with an empty body, or a
143
+ * status-only 204/205 — {@link recoverUnrenderedStatus}), and the response
144
+ * being a download the browser refuses to commit as a document
145
+ * (`Content-Disposition: attachment` — {@link recoverDownloadResponse}).
138
146
  *
139
147
  * @returns the response `goto` resolved with, plus `recovered` when the
140
148
  * navigation failure was recovered into a finished result the caller must
@@ -20,6 +20,27 @@ export interface PlaywrightResponse {
20
20
  */
21
21
  request?(): PlaywrightRequest;
22
22
  }
23
+ /**
24
+ * A download Chromium started instead of committing a document (a response
25
+ * with `Content-Disposition: attachment`).
26
+ */
27
+ export interface PlaywrightDownload {
28
+ cancel(): Promise<void>;
29
+ }
30
+ /**
31
+ * The page's `APIRequestContext` surface the download recovery refetches
32
+ * through — plain HTTP from the Playwright client, the context's cookie jar,
33
+ * and none of Chromium's render-or-download semantics. Reuses
34
+ * {@link PlaywrightResponse} because that is structurally what an
35
+ * `APIResponse` is for the members this plugin reads. Absent on fakes that
36
+ * do not exercise the recovery.
37
+ */
38
+ export interface PlaywrightRequestContext {
39
+ get(url: string, options?: {
40
+ timeout?: number;
41
+ signal?: AbortSignal;
42
+ }): Promise<PlaywrightResponse>;
43
+ }
23
44
  /** The request side of a response, for main-frame filtering. */
24
45
  export interface PlaywrightRequest {
25
46
  /** True for navigations (document loads and their redirect hops). */
@@ -54,6 +75,13 @@ export interface PlaywrightPage {
54
75
  * document). Absent on minimal fakes (content polling covers them).
55
76
  */
56
77
  on?(event: 'response', listener: (response: PlaywrightResponse) => void): unknown;
78
+ /**
79
+ * Download notification — a navigation the browser handed to its download
80
+ * manager instead of committing a document (`Content-Disposition:
81
+ * attachment`); the fetch's guard cancels it so nothing lands on disk.
82
+ * Absent on minimal fakes (the recovery does not depend on it).
83
+ */
84
+ on?(event: 'download', listener: (download: PlaywrightDownload) => void): unknown;
57
85
  /**
58
86
  * Evaluate an expression in the page — the challenge probe's live-DOM
59
87
  * path. Absent on minimal fakes (content polling covers them).
@@ -61,6 +89,11 @@ export interface PlaywrightPage {
61
89
  evaluate?(script: string, arg?: unknown): Promise<unknown>;
62
90
  /** The main frame handle; compare with `request.frame()` for filtering. */
63
91
  mainFrame?(): unknown;
92
+ /**
93
+ * The page's API request context — the download recovery's refetch path
94
+ * (see {@link PlaywrightRequestContext}). Absent on minimal fakes.
95
+ */
96
+ request?: PlaywrightRequestContext;
64
97
  }
65
98
  /**
66
99
  * A browser context: fetch-owned and isolated (local backend, or CDP
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "dsh-web-fetch-playwright",
3
- "version": "0.2.10",
3
+ "version": "0.2.11",
4
4
  "description": "Playwright/CDP web-fetch provider for DeepSeek Harness: renders pages in a real browser, denoises them (Readability + DOMPurify), and returns markdown.",
5
5
  "type": "module",
6
6
  "license": "MIT",
package/src/provider.ts CHANGED
@@ -7,11 +7,16 @@
7
7
  * an error. Chromium can make that awkward: it refuses to render status-only
8
8
  * responses (error statuses with an empty body — httpbin's `/status/N` —
9
9
  * plus 204/205), failing the whole navigation with
10
- * `net::ERR_HTTP_RESPONSE_CODE_FAILURE` / `net::ERR_ABORTED` even though the
11
- * response itself arrived. The last main-frame response is therefore tracked
12
- * on every fetch, and such a failed `goto` is recovered into the
13
- * status-bearing result the contract demands instead of a
14
- * `WEB_PROVIDER_ERROR` (see {@link recoverUnrenderedStatus}).
10
+ * `net::ERR_HTTP_RESPONSE_CODE_FAILURE` / `net::ERR_ABORTED`, and it hands a
11
+ * `Content-Disposition: attachment` response (publish.twitter.com's oembed
12
+ * JSON, the URL agents use to read a tweet) to its download manager instead
13
+ * of committing a document, failing `goto` with "Download is starting" —
14
+ * even though the response itself arrived and is decodable. The last
15
+ * main-frame response is therefore tracked on every fetch, and such failed
16
+ * `goto`s are recovered — the status-only shape into the status-bearing
17
+ * result ({@link recoverUnrenderedStatus}), the download shape via a
18
+ * refetch through the page's API request context
19
+ * ({@link recoverDownloadResponse}) — instead of a `WEB_PROVIDER_ERROR`.
15
20
  *
16
21
  * Lifecycle: the local backend launches a browser per fetch and closes it —
17
22
  * nothing outlives the call. The CDP backend keeps ONE shared connection to
@@ -55,7 +60,7 @@ import type { ResolvedConfig } from './config.ts'
55
60
  import { CdpConnectionPool } from './cdp-pool.ts'
56
61
  import { htmlToMarkdown } from './markdown.ts'
57
62
  import { resolveCdpBackend, resolvePlaywrightBackend } from './playwright-resolve.ts'
58
- import type { PlaywrightBrowser, PlaywrightContext, PlaywrightPage, PlaywrightResponse, PlaywrightRoute } from './types.ts'
63
+ import type { PlaywrightBrowser, PlaywrightContext, PlaywrightDownload, PlaywrightPage, PlaywrightResponse, PlaywrightRoute } from './types.ts'
59
64
 
60
65
  /** Stable id this provider registers under (the bundle patch pins it). */
61
66
  export const PLAYWRIGHT_FETCH_PROVIDER_ID = 'playwright'
@@ -373,6 +378,7 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
373
378
  const lease = await this.cdpPool.acquire(endpoint, timeout, effectiveContextMode(config))
374
379
  await installResourceFilter(lease.page)
375
380
  guardPopups(lease.page)
381
+ guardDownloads(lease.page)
376
382
  return {
377
383
  browser: lease.browser,
378
384
  context: lease.context,
@@ -396,6 +402,7 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
396
402
  const page = await context.newPage()
397
403
  await installResourceFilter(page)
398
404
  guardPopups(page)
405
+ guardDownloads(page)
399
406
  return { browser, context, page }
400
407
  } catch (error: unknown) {
401
408
  // A partial setup (browser launched, then newContext/newPage failed)
@@ -515,9 +522,12 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
515
522
 
516
523
  /**
517
524
  * One tracked `goto`: the navigation response on success, or — when
518
- * Chromium failed the navigation purely because of the HTTP status it
519
- * received (an error status with an empty body, or a status-only 204/205)
520
- * — the recovered status-bearing result from {@link recoverUnrenderedStatus}.
525
+ * Chromium failed the navigation for a reason that is not a transport
526
+ * error — the recovered finished result. Two such reasons are recovered:
527
+ * the HTTP status it received (an error status with an empty body, or a
528
+ * status-only 204/205 — {@link recoverUnrenderedStatus}), and the response
529
+ * being a download the browser refuses to commit as a document
530
+ * (`Content-Disposition: attachment` — {@link recoverDownloadResponse}).
521
531
  *
522
532
  * @returns the response `goto` resolved with, plus `recovered` when the
523
533
  * navigation failure was recovered into a finished result the caller must
@@ -539,6 +549,8 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
539
549
  if (!deadline.signal.aborted) {
540
550
  const recovered = await recoverUnrenderedStatus(error, tracker, url, denoise)
541
551
  if (recovered !== undefined) return { response: null, recovered }
552
+ const download = await recoverDownloadResponse(error, page, tracker, url, deadline, denoise)
553
+ if (download !== undefined) return { response: null, recovered: download }
542
554
  }
543
555
  throw error
544
556
  }
@@ -737,6 +749,104 @@ async function recoverUnrenderedStatus(
737
749
  return capResult(finalUrl, status, { kind: 'text', content: markdown })
738
750
  }
739
751
 
752
+ /**
753
+ * Whether a `goto` throw is Chromium handing the navigation to its download
754
+ * manager instead of committing a document. Playwright rejects with
755
+ * "Download is starting" when the navigation response carries
756
+ * `Content-Disposition: attachment` (publish.twitter.com's oembed endpoint —
757
+ * the workaround URL agents use to read a tweet — serves exactly that:
758
+ * decodable `application/json` the browser still refuses to render). The
759
+ * attachment disposition on the tracked response is a second,
760
+ * engine-version-proof signal for engines that report a plain `ERR_ABORTED`
761
+ * instead; note `ERR_ABORTED` alone is NOT enough (the status recovery
762
+ * above legitimately owns the 204/205 shape, and SPA cancels abort too).
763
+ *
764
+ * @param error - the value `page.goto` rejected with.
765
+ * @param response - the last tracked main-frame response, if any.
766
+ * @returns true when the navigation failed because it became a download.
767
+ */
768
+ function isDownloadFailure(error: unknown, response: PlaywrightResponse | null): boolean {
769
+ if (!(error instanceof Error)) return false
770
+ if (error.message.includes('Download is starting')) return true
771
+ if (!error.message.includes('ERR_ABORTED')) return false
772
+ const disposition = (response?.headers()['content-disposition'] ?? '').toLowerCase()
773
+ return disposition.includes('attachment')
774
+ }
775
+
776
+ /**
777
+ * Recover a download-turned navigation into the finished fetch result the
778
+ * seam's contract demands. The resource itself is often perfectly decodable
779
+ * — the oembed endpoint's `application/json` classifies as `text` — the
780
+ * browser just will not commit it as a document, and the response it tracked
781
+ * cannot yield its body either (Chromium streams a download to its download
782
+ * manager, so `Network.getResponseBody` answers "No resource with given
783
+ * identifier found"; both verified against a real browser). The body is
784
+ * therefore refetched through the page's API request context: plain HTTP
785
+ * from the Playwright client — same cookie jar, same URL, no render-or-
786
+ * download decision — which works identically on the local and CDP
787
+ * backends because it never touches the remote artifact.
788
+ *
789
+ * Undecodable downloads (a PDF, a ZIP) still fail with
790
+ * `WEB_UNSUPPORTED_CONTENT_TYPE`, classified from what the navigation
791
+ * actually served; a playwright build without a request context surfaces a
792
+ * clear "the browser will not render this" error instead of the raw
793
+ * "Download is starting".
794
+ *
795
+ * @param error - the value `page.goto` rejected with.
796
+ * @param page - the fetch's tab, for its API request context.
797
+ * @param tracker - the fetch's main-frame response record.
798
+ * @param url - the request URL (fallback when the response carries none).
799
+ * @param deadline - the fetch budget, applied to the refetch.
800
+ * @param denoise - the settings toggle, honored as on the normal HTML path.
801
+ * @returns the finished fetch result, or undefined when the failure is not
802
+ * a download navigation (the throw should propagate).
803
+ */
804
+ async function recoverDownloadResponse(
805
+ error: unknown,
806
+ page: PlaywrightPage,
807
+ tracker: MainFrameTracker,
808
+ url: URL,
809
+ deadline: Deadline,
810
+ denoise: boolean,
811
+ ): Promise<WebFetchResult | undefined> {
812
+ const tracked = tracker.last()
813
+ if (!isDownloadFailure(error, tracked)) return undefined
814
+ if (tracked !== null) {
815
+ // What the site served the navigation is the truth about the resource:
816
+ // an undecodable download (PDF/ZIP/…) has no business being refetched.
817
+ const trackedKind = classifyContentType(tracked.headers()['content-type'])
818
+ if (trackedKind === undefined) {
819
+ throw new WebError(`unsupported content type "${tracked.headers()['content-type'] ?? 'unknown'}" (served as a download)`, 'WEB_UNSUPPORTED_CONTENT_TYPE')
820
+ }
821
+ }
822
+ if (page.request === undefined) {
823
+ throw new WebError(
824
+ 'the response is a file download (Content-Disposition: attachment) the browser will not render, and this playwright build exposes no request context to refetch it',
825
+ 'WEB_PROVIDER_ERROR',
826
+ { cause: error },
827
+ )
828
+ }
829
+ const fetchUrl = tracked?.url?.() ?? url.toString()
830
+ const refetched = await page.request.get(fetchUrl, { timeout: deadline.remainingMs(), signal: deadline.signal })
831
+ const headers = refetched.headers()
832
+ // The refetch speaks for the bytes it returned; only a content-type-free
833
+ // answer falls back to what the navigation served.
834
+ const servedType = (headers['content-type'] ?? '').trim() !== '' ? headers['content-type'] : tracked?.headers()['content-type']
835
+ const kind = classifyContentType(servedType)
836
+ if (kind === undefined) {
837
+ throw new WebError(`unsupported content type "${servedType ?? 'unknown'}"`, 'WEB_UNSUPPORTED_CONTENT_TYPE')
838
+ }
839
+ const finalUrl = refetched.url?.() ?? fetchUrl
840
+ const statusCode = refetched.status()
841
+ const content = await refetched.text()
842
+ if (kind === 'text') {
843
+ return capResult(finalUrl, statusCode, { kind: 'text', content })
844
+ }
845
+ if (!denoise) return capResult(finalUrl, statusCode, { kind: 'html', content })
846
+ const { markdown } = htmlToMarkdown(content, finalUrl)
847
+ return capResult(finalUrl, statusCode, { kind: 'text', content: markdown })
848
+ }
849
+
740
850
  /**
741
851
  * Probe the live document for challenge markers. Prefers the in-page probe
742
852
  * (`page.evaluate`) — the only thing that can see SPA-style clears — and
@@ -829,6 +939,23 @@ async function installResourceFilter(owner: {
829
939
  }
830
940
  }
831
941
 
942
+ /**
943
+ * Cancel any download Chromium starts on the fetch's tab. A navigation whose
944
+ * response carries `Content-Disposition: attachment` never commits a
945
+ * document — `goto` rejects with "Download is starting" and the recovery
946
+ * refetches the bytes (see {@link recoverDownloadResponse}) — but the
947
+ * browser still hands the original body to its download manager, which in
948
+ * profile mode would drop files into the remote machine's download dir.
949
+ * Best-effort: a page without the listener just loses the guard.
950
+ */
951
+ function guardDownloads(page: PlaywrightPage): void {
952
+ try {
953
+ page.on?.('download', (download: PlaywrightDownload) => { void download.cancel().catch(() => {}) })
954
+ } catch {
955
+ // keep going without the guard
956
+ }
957
+ }
958
+
832
959
  /**
833
960
  * Close any popup a fetched page spawns so nothing outlives the fetch's tab
834
961
  * — `page.close()` does not auto-close windows the page opened, and in
package/src/types.ts CHANGED
@@ -22,6 +22,26 @@ export interface PlaywrightResponse {
22
22
  request?(): PlaywrightRequest
23
23
  }
24
24
 
25
+ /**
26
+ * A download Chromium started instead of committing a document (a response
27
+ * with `Content-Disposition: attachment`).
28
+ */
29
+ export interface PlaywrightDownload {
30
+ cancel(): Promise<void>
31
+ }
32
+
33
+ /**
34
+ * The page's `APIRequestContext` surface the download recovery refetches
35
+ * through — plain HTTP from the Playwright client, the context's cookie jar,
36
+ * and none of Chromium's render-or-download semantics. Reuses
37
+ * {@link PlaywrightResponse} because that is structurally what an
38
+ * `APIResponse` is for the members this plugin reads. Absent on fakes that
39
+ * do not exercise the recovery.
40
+ */
41
+ export interface PlaywrightRequestContext {
42
+ get(url: string, options?: { timeout?: number; signal?: AbortSignal }): Promise<PlaywrightResponse>
43
+ }
44
+
25
45
  /** The request side of a response, for main-frame filtering. */
26
46
  export interface PlaywrightRequest {
27
47
  /** True for navigations (document loads and their redirect hops). */
@@ -52,6 +72,13 @@ export interface PlaywrightPage {
52
72
  * document). Absent on minimal fakes (content polling covers them).
53
73
  */
54
74
  on?(event: 'response', listener: (response: PlaywrightResponse) => void): unknown
75
+ /**
76
+ * Download notification — a navigation the browser handed to its download
77
+ * manager instead of committing a document (`Content-Disposition:
78
+ * attachment`); the fetch's guard cancels it so nothing lands on disk.
79
+ * Absent on minimal fakes (the recovery does not depend on it).
80
+ */
81
+ on?(event: 'download', listener: (download: PlaywrightDownload) => void): unknown
55
82
  /**
56
83
  * Evaluate an expression in the page — the challenge probe's live-DOM
57
84
  * path. Absent on minimal fakes (content polling covers them).
@@ -59,6 +86,11 @@ export interface PlaywrightPage {
59
86
  evaluate?(script: string, arg?: unknown): Promise<unknown>
60
87
  /** The main frame handle; compare with `request.frame()` for filtering. */
61
88
  mainFrame?(): unknown
89
+ /**
90
+ * The page's API request context — the download recovery's refetch path
91
+ * (see {@link PlaywrightRequestContext}). Absent on minimal fakes.
92
+ */
93
+ request?: PlaywrightRequestContext
62
94
  }
63
95
 
64
96
  /**