dsh-read-url 0.4.8 → 1.0.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "dsh-read-url",
-  "version": "0.4.8",
-  "description": "DeepSeek Harness URL reader: fetch any webpage, auto-detect encoding (GBK/GB2312/UTF-8/Big5), extract clean main content, output compact text/Markdown to save tokens. Zero runtime dependencies.",
+  "version": "1.0.0",
+  "description": "DeepSeek Harness URL reader: fetch any webpage (HTML/JSON/RSS/Atom), auto-detect encoding (UTF-16 BOM/GBK/Big5/Shift-JIS), extract clean main content with auto-pagination, output compact text/Markdown to save tokens. Zero runtime dependencies, no API key, drop-in.",
   "type": "module",
   "main": "index.js",
   "files": [

package/proxy-fallback.js

@@ -15,6 +15,23 @@
 // model/user can act (turn the proxy on, or accept the network boundary).
 import { execFile, execFileSync } from 'node:child_process'
 
+// Some servers omit the Content-Type header entirely; the content-type gate
+// then has nothing to test and a binary body (PDF/PNG/zip) would flow into
+// the HTML pipeline as mojibake. Sniff the leading bytes instead: text
+// documents never contain NUL bytes or a run of control characters.
+// Shared by the direct path (index.js) and the curl proxy path below.
+export function looksBinary(buffer) {
+  const n = Math.min(buffer.length, 512)
+  if (!n) return false
+  let ctrl = 0
+  for (let i = 0; i < n; i++) {
+    const b = buffer[i]
+    if (b === 0) return true
+    if (b < 9 || (b > 13 && b < 32)) ctrl++
+  }
+  return ctrl / n > 0.3
+}
+
 let sysProxyCache = null // '' = checked & absent; string = proxy URL
 let sysProxyCheckedAt = 0 // cache timestamp; a failed read is re-tried after 60s
 const SYS_PROXY_RETRY_MS = 60 * 1000
@@ -98,8 +115,15 @@ export async function fetchViaCurlProxy(url, cfg, externalSignal, proxyOverride)
     const [codeStr, ...ctParts] = meta.split(' ')
     const code = Number(codeStr)
     const contentType = ctParts.join(' ')
+    if (code >= 200 && code < 300) {
+      // Headerless responses: gate binary bodies out (same rule as the direct
+      // path — curl writes an empty %{content_type} then).
+      if (!contentType && looksBinary(body)) {
+        return { error: 'Unsupported content-type (no header, binary body)' }
+      }
+    }
     if (!(code >= 200 && code < 300)) return { error: `HTTP ${code}` }
-    if (contentType && !/text\/html|application\/xhtml|text\/plain/i.test(contentType)) {
+    if (contentType && !/text\/html|application\/xhtml|text\/plain|\/json|[+/]xml/i.test(contentType)) {
       return { error: `Unsupported content-type: ${contentType.split(';')[0]}` }
     }
     return { buffer: body, contentType, finalUrl: url, viaProxy: proxy }

package/spa.js

@@ -18,10 +18,27 @@ export function looksLikeSpa(html) {
   return n >= 5
 }
 
+// High-specificity markers of a bot-challenge interstitial (Cloudflare and
+// friends). Measured: ruanyifeng.com served "Just a moment..." whose DOM
+// stabilizes while the challenge sits in "Verification successful. Waiting
+// for ... to respond" — the poll loop returned that transitional page and
+// the caller mistook it for an improvement over static extraction.
+const CHALLENGE_RE = /just a moment|performing security verification|verifying (you are|your) human|cf-chl|cf_chl|cdn-cgi\/challenge-platform|attention required!\s*[|·]?\s*cloudflare|enable javascript and cookies to continue/i
+
+export function looksLikeChallenge(html) {
+  return typeof html === 'string' && CHALLENGE_RE.test(html)
+}
+
 async function getBrowser() {
   if (!browserPromise) {
-    const { chromium } = await import('playwright')
-    browserPromise = chromium.launch({ headless: true })
+    // A failed launch (playwright later installed / chromium download finished)
+    // must not stay cached as a rejected promise — reset so the next call retries.
+    browserPromise = import('playwright')
+      .then(({ chromium }) => chromium.launch({ headless: true }))
+      .catch((e) => {
+        browserPromise = null
+        throw e
+      })
   }
   return browserPromise
 }
@@ -60,7 +77,21 @@ export async function renderPage(url, externalSignal) {
       if (len === prevLen && prevLen >= 0) break
       prevLen = len
     }
-    const html = await page.content()
+    // A bot-challenge interstitial "stabilizes" the same way real content
+    // does while it waits to redirect. When the DOM settled on a challenge
+    // page, keep polling briefly: the challenge often clears and navigates
+    // on its own within a few seconds (measured: Cloudflare "Verification
+    // successful. Waiting for ... to respond").
+    let html = await page.content()
+    if (looksLikeChallenge(html)) {
+      const t1 = Date.now()
+      while (Date.now() - t1 < 8000) {
+        if (externalSignal && externalSignal.aborted) return { error: 'cancelled' }
+        await page.waitForTimeout(700)
+        html = await page.content()
+        if (!looksLikeChallenge(html)) break
+      }
+    }
     const finalUrl = page.url()
     return { html, finalUrl }
   } catch (e) {
@@ -73,9 +104,13 @@ export async function renderPage(url, externalSignal) {
 
 // Called on plugin unload (temporal composability): release the browser.
 export async function closeBrowser() {
-  if (browserPromise) {
-    const b = await browserPromise
+  if (!browserPromise) return
+  const p = browserPromise
+  browserPromise = null
+  try {
+    const b = await p
     await b.close().catch(() => {})
-    browserPromise = null
+  } catch {
+    // launch never completed — nothing to close
   }
 }