dsh-web-fetch-playwright 0.2.8 → 0.2.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -34,6 +34,8 @@ web_fetch (tool-web)
34
34
  └─ Markdown (or raw HTML when denoise is off)
35
35
  ```
36
36
 
37
+ **Non-2xx statuses are results, not errors** — the web seam's contract (`@deepseek-ai/dsh-web`): a fetched 404/418/500 comes back with its status code, exactly what `dsh-web-fetch-http` does. Chromium complicates this for *status-only* responses — an error status with an empty body (httpbin's `/status/404`) fails the whole navigation with `net::ERR_HTTP_RESPONSE_CODE_FAILURE`, and a 204/205 is aborted (`net::ERR_ABORTED`) because nothing is renderable. The provider therefore tracks the last main-frame response on every fetch (that response arrives before Chromium fails the navigation) and recovers such failures into the status-bearing result, so `web_fetch` reports `Fetched … (HTTP 404)` instead of a `WEB_PROVIDER_ERROR` throw. A recovered response that is itself a Cloudflare challenge edge still fails with `WEB_FETCH_CHALLENGE`; transport errors and cancellations stay errors.
38
+
37
39
  ## Requirements
38
40
 
39
41
  - DSH web profile (`dsh web`), Node.js ≥ 20.
package/README.zh-CN.md CHANGED
@@ -34,6 +34,8 @@ web_fetch (tool-web)
34
34
  └─ Markdown(关闭降噪时返回原始 HTML)
35
35
  ```
36
36
 
37
+ **非 2xx 状态码是结果而非错误** —— 这是 web seam 的契约(`@deepseek-ai/dsh-web`):抓取到的 404/418/500 会连同状态码一起正常返回,与 `dsh-web-fetch-http` 的行为一致。但对*仅有状态*的响应 Chromium 会把事情搞砸 —— 空 body 的错误状态(httpbin 的 `/status/404`)会让整个导航以 `net::ERR_HTTP_RESPONSE_CODE_FAILURE` 失败,204/205 则因无可渲染内容被中止(`net::ERR_ABORTED`)。因此 provider 在每次抓取中都跟踪最后一个主框架响应(该响应先于 Chromium 判定导航失败到达),并把这类失败恢复成带状态码的正常结果,`web_fetch` 会报告 `Fetched … (HTTP 404)` 而不是抛出 `WEB_PROVIDER_ERROR`。若恢复出的响应本身来自 Cloudflare 挑战边缘,仍以 `WEB_FETCH_CHALLENGE` 失败;传输错误与取消也依旧按错误处理。
38
+
37
39
  ## 环境要求
38
40
 
39
41
  - DSH web profile(`dsh web`),Node.js ≥ 20。
package/lib/client.js CHANGED
@@ -412,12 +412,12 @@ window.__ModuleLoader__.load({
412
412
  document.head.appendChild(tag);
413
413
  }
414
414
  var PluginCard_module_css_default = {
415
- "footer": "UM8-Rq_footer",
415
+ "readOnly": "UM8-Rq_readOnly",
416
416
  "form": "UM8-Rq_form",
417
- "discard": "UM8-Rq_discard",
418
- "save": "UM8-Rq_save",
419
417
  "failed": "UM8-Rq_failed",
420
- "readOnly": "UM8-Rq_readOnly"
418
+ "footer": "UM8-Rq_footer",
419
+ "save": "UM8-Rq_save",
420
+ "discard": "UM8-Rq_discard"
421
421
  };
422
422
  //#endregion
423
423
  //#region src/client/PluginCard.tsx
@@ -477,26 +477,26 @@ window.__ModuleLoader__.load({
477
477
  document.head.appendChild(tag);
478
478
  }
479
479
  var fields_module_css_default = {
480
- "radioContent": "CFSaVG_radioContent",
481
- "field": "CFSaVG_field",
482
- "checkboxRow": "CFSaVG_checkboxRow",
483
- "radioHint": "CFSaVG_radioHint",
484
- "reset": "CFSaVG_reset",
485
- "badges": "CFSaVG_badges",
486
- "radioGroup": "CFSaVG_radioGroup",
487
- "fieldEmbedded": "CFSaVG_fieldEmbedded",
488
- "radioOption": "CFSaVG_radioOption",
489
- "radioLabel": "CFSaVG_radioLabel",
480
+ "input": "CFSaVG_input",
490
481
  "inputInvalid": "CFSaVG_inputInvalid",
482
+ "hintIndented": "CFSaVG_hintIndented",
491
483
  "radioText": "CFSaVG_radioText",
492
- "label": "CFSaVG_label",
484
+ "radioPick": "CFSaVG_radioPick",
485
+ "radioLabel": "CFSaVG_radioLabel",
493
486
  "invalid": "CFSaVG_invalid",
494
- "hintIndented": "CFSaVG_hintIndented",
487
+ "radioHint": "CFSaVG_radioHint",
488
+ "radioOption": "CFSaVG_radioOption",
489
+ "badges": "CFSaVG_badges",
490
+ "label": "CFSaVG_label",
491
+ "hint": "CFSaVG_hint",
495
492
  "badge": "CFSaVG_badge",
496
- "input": "CFSaVG_input",
493
+ "field": "CFSaVG_field",
494
+ "reset": "CFSaVG_reset",
495
+ "fieldEmbedded": "CFSaVG_fieldEmbedded",
496
+ "radioGroup": "CFSaVG_radioGroup",
497
+ "checkboxRow": "CFSaVG_checkboxRow",
497
498
  "head": "CFSaVG_head",
498
- "radioPick": "CFSaVG_radioPick",
499
- "hint": "CFSaVG_hint"
499
+ "radioContent": "CFSaVG_radioContent"
500
500
  };
501
501
  //#endregion
502
502
  //#region src/client/fields.tsx
package/lib/index.js CHANGED
@@ -1,14 +1,13 @@
1
- import { createRequire } from "node:module";
1
+ import { createRequire, isBuiltin } from "node:module";
2
2
  import z from "@deepseek-ai/schemastery";
3
3
  import { WebError } from "@deepseek-ai/dsh-web";
4
- import { JSDOM, VirtualConsole } from "jsdom";
5
4
  import createDOMPurify from "dompurify";
6
5
  import { Readability } from "@mozilla/readability";
7
6
  import TurndownService from "turndown";
8
7
  import { gfm } from "@joplin/turndown-plugin-gfm";
8
+ import { fileURLToPath, pathToFileURL } from "node:url";
9
9
  import { accessSync, constants, readFileSync, realpathSync, statSync } from "node:fs";
10
10
  import { delimiter, dirname, join } from "node:path";
11
- import { pathToFileURL } from "node:url";
12
11
  //#region src/config.ts
13
12
  /** Default CDP endpoint when the settings section leaves it blank. */
14
13
  const DEFAULT_CDP_ENDPOINT = "127.0.0.1:9222";
@@ -518,6 +517,87 @@ var CdpConnectionPool = class {
518
517
  }
519
518
  };
520
519
  //#endregion
520
+ //#region src/host-resolve-compat.ts
521
+ /**
522
+ * Host resolve compat for the `punycode/` builtin-slash idiom.
523
+ *
524
+ * dsh web (runtime resolution in `dsh-app-boot`) intercepts Node's CommonJS
525
+ * resolver for every package installed in a profile. Its router classifies a
526
+ * bare request by stripping one trailing path segment, which turns tr46's
527
+ * `require("punycode/")` — the standard npm-package-over-deprecated-builtin
528
+ * idiom — into the builtin name `punycode`, and `require.resolve.paths()` of a
529
+ * builtin returns `null`; the unguarded iteration then throws
530
+ * `TypeError: createRequire.resolve.paths is not a function or its return
531
+ * value is not iterable`, killing the whole `jsdom` import chain (jsdom →
532
+ * whatwg-url → tr46) and with it this plugin's activation.
533
+ *
534
+ * The shim below is installed lazily, immediately before the first `jsdom`
535
+ * load, and only when the running resolver actually exhibits the failure
536
+ * (probed with the very request that breaks). It wraps the public
537
+ * `Module._resolveFilename` — whatever it currently is, native or already
538
+ * host-intercepted — and re-dispatches exactly the affected request shape
539
+ * (`<deprecated builtin>/`) with an explicit `paths` list. An explicit
540
+ * `options.paths` is the documented bypass of every resolution interceptor in
541
+ * the chain (the host wrapper delegates straight to the native resolver for
542
+ * it), so the request lands on the real npm package — the same file plain
543
+ * Node would have found. Hosts without the bug never see the wrapper at all;
544
+ * the probe itself changes nothing.
545
+ *
546
+ * The ESM flavor of the same router path exists in theory (`import
547
+ * 'punycode/'`), but nothing in this plugin's tree imports it that way —
548
+ * tr46 is CommonJS — so the CommonJS hook is the complete fix here.
549
+ *
550
+ * @module dsh-web-fetch-playwright/host-resolve-compat
551
+ */
552
+ /** `createRequire` anchored at this module, wherever the bundle is installed. */
553
+ const require_$1 = createRequire(import.meta.url);
554
+ /** This file's own path, the anchor the probe resolves from. */
555
+ const SELF_FILENAME = fileURLToPath(import.meta.url);
556
+ /**
557
+ * Whether a request is the npm-over-builtin idiom: one bare package name with
558
+ * a single trailing slash, whose stem is a Node builtin (today, in practice:
559
+ * `punycode/`).
560
+ * @param request - the module request seen by `Module._resolveFilename`.
561
+ * @returns `true` for `punycode/`-shaped requests; relative paths, scoped
562
+ * names, subpaths, and ordinary packages all answer `false`.
563
+ */
564
+ function isBuiltinSlashRequest(request) {
565
+ if (request.length < 2 || !request.endsWith("/")) return false;
566
+ const first = request.charAt(0);
567
+ if (first === "" || "./#@".includes(first)) return false;
568
+ const stem = request.slice(0, -1);
569
+ return stem !== "" && !stem.includes("/") && isBuiltin(stem);
570
+ }
571
+ /** Probe outcome once known: `true` = shim installed or verified unneeded. */
572
+ let settled = false;
573
+ /**
574
+ * Make `require("<deprecated builtin>/")` resolvable under hosts whose
575
+ * profile-scope resolution interceptor mishandles it. Idempotent, synchronous,
576
+ * and a no-op on healthy resolvers (the probe resolves successfully and no
577
+ * wrapper is installed). Must run before the first `require('jsdom')`.
578
+ */
579
+ function installBuiltinSlashResolveCompat() {
580
+ if (settled) return;
581
+ settled = true;
582
+ const ModuleClass = require_$1("node:module").Module;
583
+ const original = ModuleClass._resolveFilename;
584
+ if (typeof original !== "function") return;
585
+ try {
586
+ original.call(ModuleClass, "punycode/", { filename: SELF_FILENAME });
587
+ return;
588
+ } catch (error) {
589
+ if (!(error instanceof TypeError)) return;
590
+ }
591
+ const searchPaths = require_$1.resolve.paths("jsdom") ?? [];
592
+ ModuleClass._resolveFilename = function patchedResolveFilename(request, parent, main, options) {
593
+ if (options?.paths === void 0 && typeof request === "string" && isBuiltinSlashRequest(request)) return original.call(ModuleClass, request, parent, main, {
594
+ ...options,
595
+ paths: searchPaths
596
+ });
597
+ return original.call(ModuleClass, request, parent, main, options);
598
+ };
599
+ }
600
+ //#endregion
521
601
  //#region src/markdown.ts
522
602
  /**
523
603
  * The denoise pipeline: rendered HTML → sanitized article HTML → markdown.
@@ -532,10 +612,33 @@ var CdpConnectionPool = class {
532
612
  * options and span-safe table rules as the shipped `dsh-tool-web` renderer,
533
613
  * so output is consistent with what `web_fetch` produces elsewhere.
534
614
  *
535
- * Pure and synchronous — unit-tested against fixture pages.
615
+ * Synchronous for its callers and unit-tested against fixture pages: jsdom
616
+ * itself is loaded lazily on the first call — and, inside a dsh profile, only
617
+ * after `installBuiltinSlashResolveCompat`, so a host resolver that
618
+ * mishandles tr46's `require("punycode/")` cannot take the plugin down.
536
619
  *
537
620
  * @module dsh-web-fetch-playwright/markdown
538
621
  */
622
+ /** `createRequire` anchored at this module, so jsdom resolves from this install. */
623
+ const require_ = createRequire(import.meta.url);
624
+ /** Loaded jsdom module, cached after the first denoise. */
625
+ let jsdom;
626
+ /**
627
+ * Load jsdom on first use, installing the host resolve compat first.
628
+ *
629
+ * Deferral serves two purposes: plugin activation never depends on jsdom's
630
+ * import chain surviving the host's module-resolution interception, and the
631
+ * compat shim — which must wrap the resolver *before* tr46's
632
+ * `require("punycode/")` runs — gets a guaranteed installation point.
633
+ * @returns the jsdom module exports.
634
+ */
635
+ function loadJsdom() {
636
+ if (jsdom === void 0) {
637
+ installBuiltinSlashResolveCompat();
638
+ jsdom = require_("jsdom");
639
+ }
640
+ return jsdom;
641
+ }
539
642
  /** Layout/noise tags DOMPurify removes outright (with their content). */
540
643
  const FORBID_TAGS = [
541
644
  "nav",
@@ -666,6 +769,7 @@ turndown.addRule("tableRowWithoutSpanExpansion", {
666
769
  * @returns the markdown and the extraction mode used.
667
770
  */
668
771
  function htmlToMarkdown(html, url) {
772
+ const { JSDOM, VirtualConsole } = loadJsdom();
669
773
  const dom = new JSDOM(html, {
670
774
  url,
671
775
  virtualConsole: new VirtualConsole()
@@ -889,7 +993,16 @@ function existsAsFile(path) {
889
993
  * The Playwright `WebFetchProvider`: renders one URL in a real browser and
890
994
  * returns it as markdown (denoised) or HTML. Mirrors `dsh-web-fetch-http`'s
891
995
  * error taxonomy (URL hygiene, abort/timeout translation, content-type
892
- * classification) so the tool layer sees the same codes from either backend.
996
+ * classification) so the tool layer sees the same codes from either backend —
997
+ * including the seam contract that a fetched non-2xx status is a RESULT, not
998
+ * an error. Chromium can make that awkward: it refuses to render status-only
999
+ * responses (error statuses with an empty body — httpbin's `/status/N` —
1000
+ * plus 204/205), failing the whole navigation with
1001
+ * `net::ERR_HTTP_RESPONSE_CODE_FAILURE` / `net::ERR_ABORTED` even though the
1002
+ * response itself arrived. The last main-frame response is therefore tracked
1003
+ * on every fetch, and such a failed `goto` is recovered into the
1004
+ * status-bearing result the contract demands instead of a
1005
+ * `WEB_PROVIDER_ERROR` (see {@link recoverUnrenderedStatus}).
893
1006
  *
894
1007
  * Lifecycle: the local backend launches a browser per fetch and closes it —
895
1008
  * nothing outlives the call. The CDP backend keeps ONE shared connection to
@@ -1212,20 +1325,19 @@ var PlaywrightFetchProvider = class {
1212
1325
  async retrieve(session, url, config, deadline) {
1213
1326
  const page = session.page;
1214
1327
  const challengeWaitMs = effectiveChallengeWaitMs(config);
1215
- const tracker = challengeWaitMs > 0 ? trackMainFrameResponses(page) : void 0;
1216
- let response = await page.goto(url.toString(), {
1217
- waitUntil: "domcontentloaded",
1218
- timeout: deadline.remainingMs()
1219
- });
1220
- tracker?.seed(response);
1328
+ const tracker = trackMainFrameResponses(page);
1329
+ const first = await this.navigate(page, url, tracker, deadline, config.denoise);
1330
+ if (first.recovered !== void 0) return first.recovered;
1331
+ let response = first.response;
1332
+ tracker.seed(response);
1221
1333
  let challengeEntryResponse = null;
1222
1334
  if (challengeWaitMs > 0) {
1223
1335
  let attemptsLeft = effectiveChallengeRetries(config) + 1;
1224
1336
  for (;;) {
1225
- const verdict = await this.verdictAfterLoad(page, tracker?.last() ?? response);
1337
+ const verdict = await this.verdictAfterLoad(page, tracker.last() ?? response);
1226
1338
  if (verdict === "blocked") throw new WebError(`the site hard-blocked this fetch at its Cloudflare edge (waiting cannot clear it): ${page.url()}`, WEB_FETCH_CHALLENGE_CODE);
1227
1339
  if (verdict !== "challenge") break;
1228
- challengeEntryResponse = tracker?.last() ?? response;
1340
+ challengeEntryResponse = tracker.last() ?? response;
1229
1341
  if (await this.waitForChallengeClear(page, deadline, challengeWaitMs)) {
1230
1342
  let chained = false;
1231
1343
  try {
@@ -1236,17 +1348,16 @@ var PlaywrightFetchProvider = class {
1236
1348
  if (!chained) break;
1237
1349
  }
1238
1350
  if (--attemptsLeft <= 0) {
1239
- const lastStatus = (tracker?.last() ?? response)?.status();
1351
+ const lastStatus = (tracker.last() ?? response)?.status();
1240
1352
  throw new WebError(`the site kept serving a Cloudflare challenge (last status ${lastStatus === void 0 ? "unknown" : String(lastStatus)}) for up to ${String(challengeWaitMs)}ms across ${String(effectiveChallengeRetries(config) + 1)} attempt(s); the browser did not clear it naturally — retry later, raise challengeWaitMs, or use a profile whose browser already holds clearance`, WEB_FETCH_CHALLENGE_CODE);
1241
1353
  }
1242
- response = await page.goto(url.toString(), {
1243
- waitUntil: "domcontentloaded",
1244
- timeout: deadline.remainingMs()
1245
- });
1246
- tracker?.seed(response);
1354
+ const retry = await this.navigate(page, url, tracker, deadline, config.denoise);
1355
+ if (retry.recovered !== void 0) return retry.recovered;
1356
+ response = retry.response;
1357
+ tracker.seed(response);
1247
1358
  }
1248
1359
  }
1249
- const finalResponse = (challengeWaitMs > 0 ? tracker?.last() : void 0) ?? response;
1360
+ const finalResponse = (challengeWaitMs > 0 ? tracker.last() : void 0) ?? response;
1250
1361
  const kind = classifyContentType(finalResponse?.headers()["content-type"]);
1251
1362
  if (kind === void 0) throw new WebError(`unsupported content type "${finalResponse?.headers()["content-type"] ?? "unknown"}"`, "WEB_UNSUPPORTED_CONTENT_TYPE");
1252
1363
  const finalUrl = page.url();
@@ -1273,6 +1384,33 @@ var PlaywrightFetchProvider = class {
1273
1384
  } : result;
1274
1385
  }
1275
1386
  /**
1387
+ * One tracked `goto`: the navigation response on success, or — when
1388
+ * Chromium failed the navigation purely because of the HTTP status it
1389
+ * received (an error status with an empty body, or a status-only 204/205)
1390
+ * — the recovered status-bearing result from {@link recoverUnrenderedStatus}.
1391
+ *
1392
+ * @returns the response `goto` resolved with, plus `recovered` when the
1393
+ * navigation failure was recovered into a finished result the caller must
1394
+ * return verbatim (the throw never escapes in that case).
1395
+ */
1396
+ async navigate(page, url, tracker, deadline, denoise) {
1397
+ try {
1398
+ return { response: await page.goto(url.toString(), {
1399
+ waitUntil: "domcontentloaded",
1400
+ timeout: deadline.remainingMs()
1401
+ }) };
1402
+ } catch (error) {
1403
+ if (!deadline.signal.aborted) {
1404
+ const recovered = await recoverUnrenderedStatus(error, tracker, url, denoise);
1405
+ if (recovered !== void 0) return {
1406
+ response: null,
1407
+ recovered
1408
+ };
1409
+ }
1410
+ throw error;
1411
+ }
1412
+ }
1413
+ /**
1276
1414
  * Whether the freshly loaded document is a Cloudflare interstitial: the
1277
1415
  * response-level signals first (the documented `cf-mitigated` header, then
1278
1416
  * the 403/503 + cloudflare fallback), then — gated behind
@@ -1359,6 +1497,79 @@ function trackMainFrameResponses(page) {
1359
1497
  };
1360
1498
  }
1361
1499
  /**
1500
+ * Whether a `goto` throw is Chromium refusing to RENDER a response it
1501
+ * received — not a transport failure. Two net errors, both documented as
1502
+ * rendering decisions rather than network ones:
1503
+ *
1504
+ * - `ERR_HTTP_RESPONSE_CODE_FAILURE` — an error status whose empty body
1505
+ * leaves nothing to commit (httpbin's `/status/N`); net_error_list.h
1506
+ * describes the code as "only used by certain APIs that interpret the
1507
+ * HTTP response itself".
1508
+ * - `ERR_ABORTED` on a status-only 204/205 — Chromium never renders those
1509
+ * documents (`NavigationRequest` aborts them), while the response is
1510
+ * final and the seam contract wants it as a result.
1511
+ *
1512
+ * @param error - the value `page.goto` rejected with.
1513
+ * @param status - the status of the last tracked main-frame response.
1514
+ * @returns true when the navigation failed for the status it received.
1515
+ */
1516
+ function isUnrenderedStatusFailure(error, status) {
1517
+ if (!(error instanceof Error)) return false;
1518
+ const message = error.message;
1519
+ if (message.includes("ERR_HTTP_RESPONSE_CODE_FAILURE")) return true;
1520
+ return message.includes("ERR_ABORTED") && (status === 204 || status === 205);
1521
+ }
1522
+ /**
1523
+ * Recover the response behind a status-only navigation failure into the
1524
+ * result the web seam's contract demands — "a successful network fetch of a
1525
+ * non-2xx response is a result, not an error: the status code is part of the
1526
+ * fetched resource state" (`@deepseek-ai/dsh-web` types). The main-frame
1527
+ * response arrives (and the tracker records it) before Chromium fails the
1528
+ * navigation, so its status, headers, and URL are recoverable; only the body
1529
+ * dies with the unrendered loader, and this failure class only fires on
1530
+ * empty bodies — so an unreadable body decodes as the empty string it is.
1531
+ *
1532
+ * Deliberately NOT recovered: genuine transport errors (they carry no
1533
+ * response), and our own deadline/abort (the caller classifies those).
1534
+ *
1535
+ * @param error - the value `page.goto` rejected with.
1536
+ * @param tracker - the fetch's main-frame response record.
1537
+ * @param url - the request URL (fallback when the response carries none).
1538
+ * @param denoise - the settings toggle, honored as on the normal HTML path.
1539
+ * @returns the finished fetch result, or undefined when the failure is not
1540
+ * a recoverable status-only navigation (the throw should propagate).
1541
+ */
1542
+ async function recoverUnrenderedStatus(error, tracker, url, denoise) {
1543
+ const response = tracker.last();
1544
+ if (response === null) return void 0;
1545
+ const status = response.status();
1546
+ if (!isUnrenderedStatusFailure(error, status)) return void 0;
1547
+ const headers = response.headers();
1548
+ if (classifyChallengeResponse(status, headers) === "challenge") throw new WebError(`the site answered with a Cloudflare challenge response (HTTP ${String(status)}) whose empty body the browser refused to render; waiting cannot clear it — retry later or use a profile whose browser already holds clearance`, WEB_FETCH_CHALLENGE_CODE);
1549
+ const kind = classifyContentType(headers["content-type"]);
1550
+ if (kind === void 0) throw new WebError(`unsupported content type "${headers["content-type"] ?? "unknown"}"`, "WEB_UNSUPPORTED_CONTENT_TYPE");
1551
+ const finalUrl = response.url?.() ?? url.toString();
1552
+ let content = "";
1553
+ try {
1554
+ content = await response.text();
1555
+ } catch {
1556
+ content = "";
1557
+ }
1558
+ if (kind === "text" || content === "") return capResult(finalUrl, status, {
1559
+ kind: "text",
1560
+ content
1561
+ });
1562
+ if (!denoise) return capResult(finalUrl, status, {
1563
+ kind: "html",
1564
+ content
1565
+ });
1566
+ const { markdown } = htmlToMarkdown(content, finalUrl);
1567
+ return capResult(finalUrl, status, {
1568
+ kind: "text",
1569
+ content: markdown
1570
+ });
1571
+ }
1572
+ /**
1362
1573
  * Probe the live document for challenge markers. Prefers the in-page probe
1363
1574
  * (`page.evaluate`) — the only thing that can see SPA-style clears — and
1364
1575
  * falls back to reading and classifying the serialized HTML when the backend
@@ -0,0 +1,38 @@
1
+ /**
2
+ * Host resolve compat for the `punycode/` builtin-slash idiom.
3
+ *
4
+ * dsh web (runtime resolution in `dsh-app-boot`) intercepts Node's CommonJS
5
+ * resolver for every package installed in a profile. Its router classifies a
6
+ * bare request by stripping one trailing path segment, which turns tr46's
7
+ * `require("punycode/")` — the standard npm-package-over-deprecated-builtin
8
+ * idiom — into the builtin name `punycode`, and `require.resolve.paths()` of a
9
+ * builtin returns `null`; the unguarded iteration then throws
10
+ * `TypeError: createRequire.resolve.paths is not a function or its return
11
+ * value is not iterable`, killing the whole `jsdom` import chain (jsdom →
12
+ * whatwg-url → tr46) and with it this plugin's activation.
13
+ *
14
+ * The shim below is installed lazily, immediately before the first `jsdom`
15
+ * load, and only when the running resolver actually exhibits the failure
16
+ * (probed with the very request that breaks). It wraps the public
17
+ * `Module._resolveFilename` — whatever it currently is, native or already
18
+ * host-intercepted — and re-dispatches exactly the affected request shape
19
+ * (`<deprecated builtin>/`) with an explicit `paths` list. An explicit
20
+ * `options.paths` is the documented bypass of every resolution interceptor in
21
+ * the chain (the host wrapper delegates straight to the native resolver for
22
+ * it), so the request lands on the real npm package — the same file plain
23
+ * Node would have found. Hosts without the bug never see the wrapper at all;
24
+ * the probe itself changes nothing.
25
+ *
26
+ * The ESM flavor of the same router path exists in theory (`import
27
+ * 'punycode/'`), but nothing in this plugin's tree imports it that way —
28
+ * tr46 is CommonJS — so the CommonJS hook is the complete fix here.
29
+ *
30
+ * @module dsh-web-fetch-playwright/host-resolve-compat
31
+ */
32
+ /**
33
+ * Make `require("<deprecated builtin>/")` resolvable under hosts whose
34
+ * profile-scope resolution interceptor mishandles it. Idempotent, synchronous,
35
+ * and a no-op on healthy resolvers (the probe resolves successfully and no
36
+ * wrapper is installed). Must run before the first `require('jsdom')`.
37
+ */
38
+ export declare function installBuiltinSlashResolveCompat(): void;
@@ -11,7 +11,10 @@
11
11
  * options and span-safe table rules as the shipped `dsh-tool-web` renderer,
12
12
  * so output is consistent with what `web_fetch` produces elsewhere.
13
13
  *
14
- * Pure and synchronous — unit-tested against fixture pages.
14
+ * Synchronous for its callers and unit-tested against fixture pages: jsdom
15
+ * itself is loaded lazily on the first call — and, inside a dsh profile, only
16
+ * after `installBuiltinSlashResolveCompat`, so a host resolver that
17
+ * mishandles tr46's `require("punycode/")` cannot take the plugin down.
15
18
  *
16
19
  * @module dsh-web-fetch-playwright/markdown
17
20
  */
@@ -2,7 +2,16 @@
2
2
  * The Playwright `WebFetchProvider`: renders one URL in a real browser and
3
3
  * returns it as markdown (denoised) or HTML. Mirrors `dsh-web-fetch-http`'s
4
4
  * error taxonomy (URL hygiene, abort/timeout translation, content-type
5
- * classification) so the tool layer sees the same codes from either backend.
5
+ * classification) so the tool layer sees the same codes from either backend —
6
+ * including the seam contract that a fetched non-2xx status is a RESULT, not
7
+ * an error. Chromium can make that awkward: it refuses to render status-only
8
+ * responses (error statuses with an empty body — httpbin's `/status/N` —
9
+ * plus 204/205), failing the whole navigation with
10
+ * `net::ERR_HTTP_RESPONSE_CODE_FAILURE` / `net::ERR_ABORTED` even though the
11
+ * response itself arrived. The last main-frame response is therefore tracked
12
+ * on every fetch, and such a failed `goto` is recovered into the
13
+ * status-bearing result the contract demands instead of a
14
+ * `WEB_PROVIDER_ERROR` (see {@link recoverUnrenderedStatus}).
6
15
  *
7
16
  * Lifecycle: the local backend launches a browser per fetch and closes it —
8
17
  * nothing outlives the call. The CDP backend keeps ONE shared connection to
@@ -121,6 +130,17 @@ export declare class PlaywrightFetchProvider implements WebFetchProvider {
121
130
  * budget before the final document is read (see the class docs).
122
131
  */
123
132
  private retrieve;
133
+ /**
134
+ * One tracked `goto`: the navigation response on success, or — when
135
+ * Chromium failed the navigation purely because of the HTTP status it
136
+ * received (an error status with an empty body, or a status-only 204/205)
137
+ * — the recovered status-bearing result from {@link recoverUnrenderedStatus}.
138
+ *
139
+ * @returns the response `goto` resolved with, plus `recovered` when the
140
+ * navigation failure was recovered into a finished result the caller must
141
+ * return verbatim (the throw never escapes in that case).
142
+ */
143
+ private navigate;
124
144
  /**
125
145
  * Whether the freshly loaded document is a Cloudflare interstitial: the
126
146
  * response-level signals first (the documented `cf-mitigated` header, then
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "dsh-web-fetch-playwright",
3
- "version": "0.2.8",
3
+ "version": "0.2.10",
4
4
  "description": "Playwright/CDP web-fetch provider for DeepSeek Harness: renders pages in a real browser, denoises them (Readability + DOMPurify), and returns markdown.",
5
5
  "type": "module",
6
6
  "license": "MIT",
@@ -0,0 +1,102 @@
1
+ /**
2
+ * Host resolve compat for the `punycode/` builtin-slash idiom.
3
+ *
4
+ * dsh web (runtime resolution in `dsh-app-boot`) intercepts Node's CommonJS
5
+ * resolver for every package installed in a profile. Its router classifies a
6
+ * bare request by stripping one trailing path segment, which turns tr46's
7
+ * `require("punycode/")` — the standard npm-package-over-deprecated-builtin
8
+ * idiom — into the builtin name `punycode`, and `require.resolve.paths()` of a
9
+ * builtin returns `null`; the unguarded iteration then throws
10
+ * `TypeError: createRequire.resolve.paths is not a function or its return
11
+ * value is not iterable`, killing the whole `jsdom` import chain (jsdom →
12
+ * whatwg-url → tr46) and with it this plugin's activation.
13
+ *
14
+ * The shim below is installed lazily, immediately before the first `jsdom`
15
+ * load, and only when the running resolver actually exhibits the failure
16
+ * (probed with the very request that breaks). It wraps the public
17
+ * `Module._resolveFilename` — whatever it currently is, native or already
18
+ * host-intercepted — and re-dispatches exactly the affected request shape
19
+ * (`<deprecated builtin>/`) with an explicit `paths` list. An explicit
20
+ * `options.paths` is the documented bypass of every resolution interceptor in
21
+ * the chain (the host wrapper delegates straight to the native resolver for
22
+ * it), so the request lands on the real npm package — the same file plain
23
+ * Node would have found. Hosts without the bug never see the wrapper at all;
24
+ * the probe itself changes nothing.
25
+ *
26
+ * The ESM flavor of the same router path exists in theory (`import
27
+ * 'punycode/'`), but nothing in this plugin's tree imports it that way —
28
+ * tr46 is CommonJS — so the CommonJS hook is the complete fix here.
29
+ *
30
+ * @module dsh-web-fetch-playwright/host-resolve-compat
31
+ */
32
+
33
+ import { createRequire, isBuiltin } from 'node:module'
34
+ import { fileURLToPath } from 'node:url'
35
+
36
+ /** `createRequire` anchored at this module, wherever the bundle is installed. */
37
+ const require_ = createRequire(import.meta.url)
38
+
39
+ /** This file's own path, the anchor the probe resolves from. */
40
+ const SELF_FILENAME = fileURLToPath(import.meta.url)
41
+
42
+ /**
43
+ * Whether a request is the npm-over-builtin idiom: one bare package name with
44
+ * a single trailing slash, whose stem is a Node builtin (today, in practice:
45
+ * `punycode/`).
46
+ * @param request - the module request seen by `Module._resolveFilename`.
47
+ * @returns `true` for `punycode/`-shaped requests; relative paths, scoped
48
+ * names, subpaths, and ordinary packages all answer `false`.
49
+ */
50
+ function isBuiltinSlashRequest(request: string): boolean {
51
+ if (request.length < 2 || !request.endsWith('/')) return false
52
+ const first = request.charAt(0)
53
+ if (first === '' || './#@'.includes(first)) return false
54
+ const stem = request.slice(0, -1)
55
+ return stem !== '' && !stem.includes('/') && isBuiltin(stem)
56
+ }
57
+
58
+ /** Probe outcome once known: `true` = shim installed or verified unneeded. */
59
+ let settled = false
60
+
61
+ /**
62
+ * Make `require("<deprecated builtin>/")` resolvable under hosts whose
63
+ * profile-scope resolution interceptor mishandles it. Idempotent, synchronous,
64
+ * and a no-op on healthy resolvers (the probe resolves successfully and no
65
+ * wrapper is installed). Must run before the first `require('jsdom')`.
66
+ */
67
+ export function installBuiltinSlashResolveCompat(): void {
68
+ if (settled) return
69
+ settled = true
70
+ const ModuleClass = require_('node:module').Module as {
71
+ _resolveFilename?: (request: string, parent: unknown, main?: boolean, options?: { paths?: string[] }) => string
72
+ }
73
+ const original = ModuleClass._resolveFilename
74
+ if (typeof original !== 'function') return
75
+
76
+ // Probe with the exact request that breaks. Resolving it (or failing with
77
+ // plain MODULE_NOT_FOUND on a tree that lacks the npm twin — where a
78
+ // wrapper could not help either) means the host needs no help.
79
+ try {
80
+ original.call(ModuleClass, 'punycode/', { filename: SELF_FILENAME })
81
+ return
82
+ } catch (error) {
83
+ if (!(error instanceof TypeError)) return
84
+ /* the compat case: install the wrapper below */
85
+ }
86
+
87
+ // The anchor list a native lookup would use from this module's position:
88
+ // this plugin's nested node_modules first, then the profile's hoisted root
89
+ // where `punycode` actually lives in a pnpm install.
90
+ const searchPaths = require_.resolve.paths('jsdom') ?? []
91
+
92
+ ModuleClass._resolveFilename = function patchedResolveFilename(
93
+ this: unknown, request: string, parent: unknown, main?: boolean, options?: { paths?: string[] },
94
+ ): string {
95
+ if (options?.paths === undefined && typeof request === 'string' && isBuiltinSlashRequest(request)) {
96
+ // `paths` set → every layer in the chain (including the host's) defers
97
+ // to the native resolver against exactly these anchors.
98
+ return original.call(ModuleClass, request, parent, main, { ...options, paths: searchPaths })
99
+ }
100
+ return original.call(ModuleClass, request, parent, main, options)
101
+ }
102
+ }
package/src/markdown.ts CHANGED
@@ -11,16 +11,46 @@
11
11
  * options and span-safe table rules as the shipped `dsh-tool-web` renderer,
12
12
  * so output is consistent with what `web_fetch` produces elsewhere.
13
13
  *
14
- * Pure and synchronous — unit-tested against fixture pages.
14
+ * Synchronous for its callers and unit-tested against fixture pages: jsdom
15
+ * itself is loaded lazily on the first call — and, inside a dsh profile, only
16
+ * after `installBuiltinSlashResolveCompat`, so a host resolver that
17
+ * mishandles tr46's `require("punycode/")` cannot take the plugin down.
15
18
  *
16
19
  * @module dsh-web-fetch-playwright/markdown
17
20
  */
18
21
 
19
- import { JSDOM, VirtualConsole } from 'jsdom'
22
+ import { createRequire } from 'node:module'
20
23
  import createDOMPurify from 'dompurify'
21
24
  import { Readability } from '@mozilla/readability'
22
25
  import TurndownService from 'turndown'
23
26
  import { gfm } from '@joplin/turndown-plugin-gfm'
27
+ import { installBuiltinSlashResolveCompat } from './host-resolve-compat.ts'
28
+
29
+ /** `createRequire` anchored at this module, so jsdom resolves from this install. */
30
+ const require_ = createRequire(import.meta.url)
31
+
32
+ /** The lazily loaded jsdom exports; jsdom is CommonJS, so loading stays synchronous. */
33
+ type JsdomModule = typeof import('jsdom')
34
+
35
+ /** Loaded jsdom module, cached after the first denoise. */
36
+ let jsdom: JsdomModule | undefined
37
+
38
+ /**
39
+ * Load jsdom on first use, installing the host resolve compat first.
40
+ *
41
+ * Deferral serves two purposes: plugin activation never depends on jsdom's
42
+ * import chain surviving the host's module-resolution interception, and the
43
+ * compat shim — which must wrap the resolver *before* tr46's
44
+ * `require("punycode/")` runs — gets a guaranteed installation point.
45
+ * @returns the jsdom module exports.
46
+ */
47
+ function loadJsdom(): JsdomModule {
48
+ if (jsdom === undefined) {
49
+ installBuiltinSlashResolveCompat()
50
+ jsdom = require_('jsdom') as JsdomModule
51
+ }
52
+ return jsdom
53
+ }
24
54
 
25
55
  /** Layout/noise tags DOMPurify removes outright (with their content). */
26
56
  const FORBID_TAGS = [
@@ -157,6 +187,7 @@ export interface DenoiseResult {
157
187
  * @returns the markdown and the extraction mode used.
158
188
  */
159
189
  export function htmlToMarkdown(html: string, url: string): DenoiseResult {
190
+ const { JSDOM, VirtualConsole } = loadJsdom()
160
191
  // jsdom's virtual console defaults to forwarding parse noise; a silent one
161
192
  // keeps broken inline CSS on random pages out of host logs.
162
193
  const dom = new JSDOM(html, { url, virtualConsole: new VirtualConsole() })
package/src/provider.ts CHANGED
@@ -2,7 +2,16 @@
2
2
  * The Playwright `WebFetchProvider`: renders one URL in a real browser and
3
3
  * returns it as markdown (denoised) or HTML. Mirrors `dsh-web-fetch-http`'s
4
4
  * error taxonomy (URL hygiene, abort/timeout translation, content-type
5
- * classification) so the tool layer sees the same codes from either backend.
5
+ * classification) so the tool layer sees the same codes from either backend —
6
+ * including the seam contract that a fetched non-2xx status is a RESULT, not
7
+ * an error. Chromium can make that awkward: it refuses to render status-only
8
+ * responses (error statuses with an empty body — httpbin's `/status/N` —
9
+ * plus 204/205), failing the whole navigation with
10
+ * `net::ERR_HTTP_RESPONSE_CODE_FAILURE` / `net::ERR_ABORTED` even though the
11
+ * response itself arrived. The last main-frame response is therefore tracked
12
+ * on every fetch, and such a failed `goto` is recovered into the
13
+ * status-bearing result the contract demands instead of a
14
+ * `WEB_PROVIDER_ERROR` (see {@link recoverUnrenderedStatus}).
6
15
  *
7
16
  * Lifecycle: the local backend launches a browser per fetch and closes it —
8
17
  * nothing outlives the call. The CDP backend keeps ONE shared connection to
@@ -414,18 +423,23 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
414
423
  ): Promise<WebFetchResult> {
415
424
  const page = session.page
416
425
  const challengeWaitMs = effectiveChallengeWaitMs(config)
426
+ // The tracker is always on: the challenge wait needs the LAST main-frame
427
+ // response (feature switch below only gates the waiting), and status
428
+ // recovery needs whatever response arrived before a failed goto.
429
+ const tracker = trackMainFrameResponses(page)
417
430
  // Feature switch: 0 keeps the exact legacy (pre-0.2.5) behavior — the
418
431
  // first response decides, no waiting — an escape hatch and the A/B
419
432
  // baseline every test proves the bug against.
420
- const tracker = challengeWaitMs > 0 ? trackMainFrameResponses(page) : undefined
421
- let response = await page.goto(url.toString(), { waitUntil: 'domcontentloaded', timeout: deadline.remainingMs() })
422
- tracker?.seed(response)
433
+ const first = await this.navigate(page, url, tracker, deadline, config.denoise)
434
+ if (first.recovered !== undefined) return first.recovered
435
+ let response = first.response
436
+ tracker.seed(response)
423
437
 
424
438
  let challengeEntryResponse: PlaywrightResponse | null = null
425
439
  if (challengeWaitMs > 0) {
426
440
  let attemptsLeft = effectiveChallengeRetries(config) + 1
427
441
  for (;;) {
428
- const verdict = await this.verdictAfterLoad(page, tracker?.last() ?? response)
442
+ const verdict = await this.verdictAfterLoad(page, tracker.last() ?? response)
429
443
  if (verdict === 'blocked') {
430
444
  throw new WebError(
431
445
  `the site hard-blocked this fetch at its Cloudflare edge (waiting cannot clear it): ${page.url()}`,
@@ -433,7 +447,7 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
433
447
  )
434
448
  }
435
449
  if (verdict !== 'challenge') break
436
- challengeEntryResponse = tracker?.last() ?? response
450
+ challengeEntryResponse = tracker.last() ?? response
437
451
  const cleared = await this.waitForChallengeClear(page, deadline, challengeWaitMs)
438
452
  if (cleared) {
439
453
  // The settled document may be the next round of a CHAINED
@@ -446,7 +460,7 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
446
460
  if (!chained) break
447
461
  }
448
462
  if (--attemptsLeft <= 0) {
449
- const lastStatus = (tracker?.last() ?? response)?.status()
463
+ const lastStatus = (tracker.last() ?? response)?.status()
450
464
  throw new WebError(
451
465
  `the site kept serving a Cloudflare challenge (last status ${lastStatus === undefined ? 'unknown' : String(lastStatus)}) for up to ${String(challengeWaitMs)}ms across ${String(effectiveChallengeRetries(config) + 1)} attempt(s); the browser did not clear it naturally — retry later, raise challengeWaitMs, or use a profile whose browser already holds clearance`,
452
466
  WEB_FETCH_CHALLENGE_CODE,
@@ -455,14 +469,16 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
455
469
  // Same page, same context — after an expired window OR a chained
456
470
  // round: any clearance cookies already earned stay in the jar for
457
471
  // this one retry, then everything is torn down as usual.
458
- response = await page.goto(url.toString(), { waitUntil: 'domcontentloaded', timeout: deadline.remainingMs() })
459
- tracker?.seed(response)
472
+ const retry = await this.navigate(page, url, tracker, deadline, config.denoise)
473
+ if (retry.recovered !== undefined) return retry.recovered
474
+ response = retry.response
475
+ tracker.seed(response)
460
476
  }
461
477
  }
462
478
 
463
479
  // Final document: the LAST main-frame response when the challenge wait
464
480
  // ran (the real page reloads in), else the response goto returned.
465
- const finalResponse = (challengeWaitMs > 0 ? tracker?.last() : undefined) ?? response
481
+ const finalResponse = (challengeWaitMs > 0 ? tracker.last() : undefined) ?? response
466
482
  const kind = classifyContentType(finalResponse?.headers()['content-type'])
467
483
  if (kind === undefined) {
468
484
  throw new WebError(`unsupported content type "${finalResponse?.headers()['content-type'] ?? 'unknown'}"`, 'WEB_UNSUPPORTED_CONTENT_TYPE')
@@ -497,6 +513,37 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
497
513
  : result
498
514
  }
499
515
 
516
+ /**
517
+ * One tracked `goto`: the navigation response on success, or — when
518
+ * Chromium failed the navigation purely because of the HTTP status it
519
+ * received (an error status with an empty body, or a status-only 204/205)
520
+ * — the recovered status-bearing result from {@link recoverUnrenderedStatus}.
521
+ *
522
+ * @returns the response `goto` resolved with, plus `recovered` when the
523
+ * navigation failure was recovered into a finished result the caller must
524
+ * return verbatim (the throw never escapes in that case).
525
+ */
526
+ private async navigate(
527
+ page: PlaywrightPage,
528
+ url: URL,
529
+ tracker: MainFrameTracker,
530
+ deadline: Deadline,
531
+ denoise: boolean,
532
+ ): Promise<{ response: PlaywrightResponse | null; recovered?: WebFetchResult }> {
533
+ try {
534
+ const response = await page.goto(url.toString(), { waitUntil: 'domcontentloaded', timeout: deadline.remainingMs() })
535
+ return { response }
536
+ } catch (error) {
537
+ // Our own deadline/abort closing the page mid-navigation is a
538
+ // cancellation (translateError classifies it) — never a status fact.
539
+ if (!deadline.signal.aborted) {
540
+ const recovered = await recoverUnrenderedStatus(error, tracker, url, denoise)
541
+ if (recovered !== undefined) return { response: null, recovered }
542
+ }
543
+ throw error
544
+ }
545
+ }
546
+
500
547
  /**
501
548
  * Whether the freshly loaded document is a Cloudflare interstitial: the
502
549
  * response-level signals first (the documented `cf-mitigated` header, then
@@ -608,6 +655,88 @@ function trackMainFrameResponses(page: PlaywrightPage): MainFrameTracker {
608
655
  }
609
656
  }
610
657
 
658
+ /**
659
+ * Whether a `goto` throw is Chromium refusing to RENDER a response it
660
+ * received — not a transport failure. Two net errors, both documented as
661
+ * rendering decisions rather than network ones:
662
+ *
663
+ * - `ERR_HTTP_RESPONSE_CODE_FAILURE` — an error status whose empty body
664
+ * leaves nothing to commit (httpbin's `/status/N`); net_error_list.h
665
+ * describes the code as "only used by certain APIs that interpret the
666
+ * HTTP response itself".
667
+ * - `ERR_ABORTED` on a status-only 204/205 — Chromium never renders those
668
+ * documents (`NavigationRequest` aborts them), while the response is
669
+ * final and the seam contract wants it as a result.
670
+ *
671
+ * @param error - the value `page.goto` rejected with.
672
+ * @param status - the status of the last tracked main-frame response.
673
+ * @returns true when the navigation failed for the status it received.
674
+ */
675
+ function isUnrenderedStatusFailure(error: unknown, status: number): boolean {
676
+ if (!(error instanceof Error)) return false
677
+ const message = error.message
678
+ if (message.includes('ERR_HTTP_RESPONSE_CODE_FAILURE')) return true
679
+ return message.includes('ERR_ABORTED') && (status === 204 || status === 205)
680
+ }
681
+
682
+ /**
683
+ * Recover the response behind a status-only navigation failure into the
684
+ * result the web seam's contract demands — "a successful network fetch of a
685
+ * non-2xx response is a result, not an error: the status code is part of the
686
+ * fetched resource state" (`@deepseek-ai/dsh-web` types). The main-frame
687
+ * response arrives (and the tracker records it) before Chromium fails the
688
+ * navigation, so its status, headers, and URL are recoverable; only the body
689
+ * dies with the unrendered loader, and this failure class only fires on
690
+ * empty bodies — so an unreadable body decodes as the empty string it is.
691
+ *
692
+ * Deliberately NOT recovered: genuine transport errors (they carry no
693
+ * response), and our own deadline/abort (the caller classifies those).
694
+ *
695
+ * @param error - the value `page.goto` rejected with.
696
+ * @param tracker - the fetch's main-frame response record.
697
+ * @param url - the request URL (fallback when the response carries none).
698
+ * @param denoise - the settings toggle, honored as on the normal HTML path.
699
+ * @returns the finished fetch result, or undefined when the failure is not
700
+ * a recoverable status-only navigation (the throw should propagate).
701
+ */
702
+ async function recoverUnrenderedStatus(
703
+ error: unknown,
704
+ tracker: MainFrameTracker,
705
+ url: URL,
706
+ denoise: boolean,
707
+ ): Promise<WebFetchResult | undefined> {
708
+ const response = tracker.last()
709
+ if (response === null) return undefined
710
+ const status = response.status()
711
+ if (!isUnrenderedStatusFailure(error, status)) return undefined
712
+ const headers = response.headers()
713
+ // A challenge edge answering with nothing to render cannot be waited out
714
+ // (no document ever loaded to clear); keep the plugin's invariant of
715
+ // never passing challenge responses through as content.
716
+ if (classifyChallengeResponse(status, headers) === 'challenge') {
717
+ throw new WebError(
718
+ `the site answered with a Cloudflare challenge response (HTTP ${String(status)}) whose empty body the browser refused to render; waiting cannot clear it — retry later or use a profile whose browser already holds clearance`,
719
+ WEB_FETCH_CHALLENGE_CODE,
720
+ )
721
+ }
722
+ const kind = classifyContentType(headers['content-type'])
723
+ if (kind === undefined) {
724
+ throw new WebError(`unsupported content type "${headers['content-type'] ?? 'unknown'}"`, 'WEB_UNSUPPORTED_CONTENT_TYPE')
725
+ }
726
+ const finalUrl = response.url?.() ?? url.toString()
727
+ // The unrenderable loader may refuse the body read too; this failure
728
+ // class only fires on empty bodies, so '' is the honest fallback.
729
+ let content = ''
730
+ try { content = await response.text() } catch { content = '' }
731
+ if (kind === 'text' || content === '') {
732
+ // Empty HTML needs no pipeline — the empty string IS the body.
733
+ return capResult(finalUrl, status, { kind: 'text', content })
734
+ }
735
+ if (!denoise) return capResult(finalUrl, status, { kind: 'html', content })
736
+ const { markdown } = htmlToMarkdown(content, finalUrl)
737
+ return capResult(finalUrl, status, { kind: 'text', content: markdown })
738
+ }
739
+
611
740
  /**
612
741
  * Probe the live document for challenge markers. Prefers the in-page probe
613
742
  * (`page.evaluate`) — the only thing that can see SPA-style clears — and