@rjshrjndrn/pi-fetch 0.1.0 → 0.1.2

package/README.md

@@ -8,7 +8,7 @@ Powered by [Defuddle](https://github.com/kepano/defuddle) by [Steph Ango](https:
 
 Registers a `web_fetch` tool that the LLM can call with any URL. Under the hood:
 
-1. **[JSDOM](https://github.com/jsdom/jsdom)** fetches and parses the HTML (lightweight, no browser engine)
+1. **Native `fetch`** retrieves the HTML (lightweight, no browser engine)
 2. **[Defuddle](https://github.com/kepano/defuddle)** extracts the main content, stripping navigation, ads, sidebars, cookie banners, and clutter
 3. Returns clean **Markdown** with metadata (title, author, description, word count)
 
@@ -32,7 +32,10 @@ Output is automatically truncated to stay within pi's context limits.
 
 ```bash
 # As a pi package
-pi install npm:pi-fetch
+pi install npm:@rjshrjndrn/pi-fetch
+
+# Or test without installing
+pi -e npm:@rjshrjndrn/pi-fetch
 
 # Or test locally
 pi -e ./extensions/index.ts
@@ -50,7 +53,7 @@ Or the LLM will use `web_fetch` automatically when it needs to read a webpage.
 
 ## Limitations
 
-- **No JavaScript rendering** — JSDOM does not execute JavaScript. SPAs that require JS to render content will return empty or minimal results. For those, you'll still need a headless browser.
+- **No JavaScript rendering** — uses native `fetch`, not a browser. SPAs that require JS to render content will return empty or minimal results. For those, you'll still need a headless browser.
 - **Some sites block non-browser requests** — sites with aggressive bot detection may reject the request.
 - **Output truncation** — very large pages are truncated to 50KB / 2000 lines to protect context window.
 
@@ -65,7 +68,6 @@ npm test
 ## Credits
 
 - **[Defuddle](https://github.com/kepano/defuddle)** by [Steph Ango (kepano)](https://github.com/kepano) — content extraction engine
-- **[JSDOM](https://github.com/jsdom/jsdom)** — HTML parsing without a browser
 - **[pi](https://github.com/badlogic/pi-mono)** by [Mario Zechner (badlogic)](https://github.com/badlogic) — agent framework
 
 ## License

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rjshrjndrn/pi-fetch",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "description": "Web content extraction for pi — fetch any URL as clean Markdown using Defuddle",
   "keywords": [
     "pi-package",
@@ -18,13 +18,15 @@
     ]
   },
   "dependencies": {
-    "defuddle": "^0.14.0"
+    "defuddle": "^0.14.0",
+    "jsdom": "^29.0.1"
   },
   "peerDependencies": {
     "@mariozechner/pi-coding-agent": "*",
     "@sinclair/typebox": "*"
   },
   "devDependencies": {
+    "@types/jsdom": "^28.0.1",
     "@types/node": "^20.0.0",
     "typescript": "^5.4.0",
     "vitest": "^1.6.0"

package/src/fetch.ts

@@ -1,3 +1,5 @@
+import { JSDOM } from "jsdom"
+
 export interface FetchResult {
   url: string
   title: string | null
@@ -10,13 +12,32 @@ export interface FetchResult {
 }
 
 const USER_AGENT =
-  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36"
+
+/**
+ * Browser-like request headers to reduce 403 blocks from bot detection.
+ */
+const BROWSER_HEADERS: Record<string, string> = {
+  "User-Agent": USER_AGENT,
+  "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
+  "Accept-Language": "en-US,en;q=0.9",
+  "Accept-Encoding": "gzip, deflate, br",
+  "Cache-Control": "no-cache",
+  "Sec-Fetch-Dest": "document",
+  "Sec-Fetch-Mode": "navigate",
+  "Sec-Fetch-Site": "none",
+  "Sec-Fetch-User": "?1",
+  "Upgrade-Insecure-Requests": "1",
+}
 
 /**
  * Fetch a URL and extract its main content as Markdown via Defuddle.
  *
- * Uses native fetch + Defuddle's Node API (no headless browser) to
+ * Uses native fetch + JSDOM + Defuddle (no headless browser) to
  * strip navigation, ads, sidebars, and clutter.
+ *
+ * JSDOM provides full DOM API support (including getComputedStyle)
+ * which Defuddle needs for accurate hidden-element detection.
  */
 export async function fetchPage(
   url: string,
@@ -26,10 +47,25 @@ export async function fetchPage(
 
   let html: string
   try {
-    const response = await fetch(url, {
+    let response = await fetch(url, {
       signal,
-      headers: { "User-Agent": USER_AGENT },
+      redirect: "follow",
+      headers: BROWSER_HEADERS,
     })
+
+    // Cloudflare blocks requests that fake a browser UA but fail the TLS
+    // fingerprint check. Retrying with an honest UA often passes through.
+    if (
+      response.status === 403 &&
+      response.headers.get("cf-mitigated") === "challenge"
+    ) {
+      response = await fetch(url, {
+        signal,
+        redirect: "follow",
+        headers: { ...BROWSER_HEADERS, "User-Agent": "pi-fetch" },
+      })
+    }
+
     if (!response.ok) {
       throw new Error(`HTTP ${response.status} ${response.statusText}`)
     }
@@ -39,7 +75,8 @@ export async function fetchPage(
     throw new Error(`Failed to fetch ${url}: ${err?.message ?? err}`)
   }
 
-  const result = await Defuddle(html, url, { markdown: true })
+  const dom = new JSDOM(html, { url, pretendToBeVisual: true })
+  const result = await Defuddle(dom.window.document, url, { markdown: true })
 
   return {
     url,