@rjshrjndrn/pi-fetch 0.1.1 → 0.1.2

package/README.md

@@ -34,6 +34,9 @@ Output is automatically truncated to stay within pi's context limits.
 # As a pi package
 pi install npm:@rjshrjndrn/pi-fetch
 
+# Or test without installing
+pi -e npm:@rjshrjndrn/pi-fetch
+
 # Or test locally
 pi -e ./extensions/index.ts
 ```

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rjshrjndrn/pi-fetch",
-  "version": "0.1.1",
+  "version": "0.1.2",
   "description": "Web content extraction for pi — fetch any URL as clean Markdown using Defuddle",
   "keywords": [
     "pi-package",
@@ -18,13 +18,15 @@
     ]
   },
   "dependencies": {
-    "defuddle": "^0.14.0"
+    "defuddle": "^0.14.0",
+    "jsdom": "^29.0.1"
   },
   "peerDependencies": {
     "@mariozechner/pi-coding-agent": "*",
     "@sinclair/typebox": "*"
   },
   "devDependencies": {
+    "@types/jsdom": "^28.0.1",
     "@types/node": "^20.0.0",
     "typescript": "^5.4.0",
     "vitest": "^1.6.0"

package/src/fetch.ts

@@ -1,3 +1,5 @@
+import { JSDOM } from "jsdom"
+
 export interface FetchResult {
   url: string
   title: string | null
@@ -10,13 +12,32 @@ export interface FetchResult {
 }
 
 const USER_AGENT =
-  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36"
+
+/**
+ * Browser-like request headers to reduce 403 blocks from bot detection.
+ */
+const BROWSER_HEADERS: Record<string, string> = {
+  "User-Agent": USER_AGENT,
+  "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
+  "Accept-Language": "en-US,en;q=0.9",
+  "Accept-Encoding": "gzip, deflate, br",
+  "Cache-Control": "no-cache",
+  "Sec-Fetch-Dest": "document",
+  "Sec-Fetch-Mode": "navigate",
+  "Sec-Fetch-Site": "none",
+  "Sec-Fetch-User": "?1",
+  "Upgrade-Insecure-Requests": "1",
+}
 
 /**
  * Fetch a URL and extract its main content as Markdown via Defuddle.
  *
- * Uses native fetch + Defuddle's Node API (no headless browser) to
+ * Uses native fetch + JSDOM + Defuddle (no headless browser) to
  * strip navigation, ads, sidebars, and clutter.
+ *
+ * JSDOM provides full DOM API support (including getComputedStyle)
+ * which Defuddle needs for accurate hidden-element detection.
  */
 export async function fetchPage(
   url: string,
@@ -26,10 +47,25 @@ export async function fetchPage(
 
   let html: string
   try {
-    const response = await fetch(url, {
+    let response = await fetch(url, {
       signal,
-      headers: { "User-Agent": USER_AGENT },
+      redirect: "follow",
+      headers: BROWSER_HEADERS,
     })
+
+    // Cloudflare blocks requests that fake a browser UA but fail the TLS
+    // fingerprint check. Retrying with an honest UA often passes through.
+    if (
+      response.status === 403 &&
+      response.headers.get("cf-mitigated") === "challenge"
+    ) {
+      response = await fetch(url, {
+        signal,
+        redirect: "follow",
+        headers: { ...BROWSER_HEADERS, "User-Agent": "pi-fetch" },
+      })
+    }
+
     if (!response.ok) {
       throw new Error(`HTTP ${response.status} ${response.statusText}`)
     }
@@ -39,7 +75,8 @@ export async function fetchPage(
     throw new Error(`Failed to fetch ${url}: ${err?.message ?? err}`)
   }
 
-  const result = await Defuddle(html, url, { markdown: true })
+  const dom = new JSDOM(html, { url, pretendToBeVisual: true })
+  const result = await Defuddle(dom.window.document, url, { markdown: true })
 
   return {
     url,