dsh-read-url 1.7.0 → 1.8.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dsh-read-url",
-  "version": "1.7.0",
+  "version": "1.8.1",
   "description": "DeepSeek Harness URL reader: fetch any webpage (HTML/JSON/RSS/Atom), auto-detect encoding (UTF-16 BOM/GBK/Big5/Shift-JIS), extract clean main content with auto-pagination, output compact text/Markdown to save tokens. Zero runtime dependencies, no API key, drop-in. Follows meta-refresh shells, reads JSON-LD metadata, honors base href.",
   "type": "module",
   "main": "index.js",

package/proxy-fallback.js

@@ -23,6 +23,19 @@ import { execFile, execFileSync } from 'node:child_process'
 export function looksBinary(buffer) {
   const n = Math.min(buffer.length, 512)
   if (!n) return false
+  const utf16le = buffer[0] === 0xff && buffer[1] === 0xfe
+  const utf16be = buffer[0] === 0xfe && buffer[1] === 0xff
+  if ((utf16le || utf16be) && buffer.length % 2 === 0) {
+    let controls = 0
+    let units = 0
+    for (let i = 2; i + 1 < n; i += 2) {
+      const code = utf16le ? buffer[i] | (buffer[i + 1] << 8) : (buffer[i] << 8) | buffer[i + 1]
+      if (code === 0) return true
+      if (code < 9 || (code > 13 && code < 32)) controls++
+      units++
+    }
+    return units > 0 && controls / units > 0.3
+  }
   let ctrl = 0
   for (let i = 0; i < n; i++) {
     const b = buffer[i]
@@ -32,6 +45,14 @@ export function looksBinary(buffer) {
   return ctrl / n > 0.3
 }
 
+// Shared by direct and proxy retrieval so a document's route cannot change
+// which text formats are accepted. Structured suffixes include vendor APIs.
+export function isFetchableContentType(contentType) {
+  const mime = (contentType || '').split(';', 1)[0].trim().toLowerCase()
+  return ['text/html', 'application/xhtml+xml', 'text/plain', 'text/markdown', 'text/csv', 'application/json', 'text/json', 'application/xml', 'text/xml'].includes(mime) ||
+    /^[a-z][a-z0-9!#$&^_.+-]{0,63}\/(?:[a-z0-9!#$&^_.+-]{1,127}\+)?(?:json|xml)$/.test(mime)
+}
+
 let sysProxyCache = null // '' = checked & absent; string = proxy URL
 let sysProxyCheckedAt = 0 // cache timestamp; a failed read is re-tried after 60s
 const SYS_PROXY_RETRY_MS = 60 * 1000
@@ -129,7 +150,7 @@ export async function fetchViaCurlProxy(url, cfg, externalSignal, proxyOverride)
       }
     }
     if (!(code >= 200 && code < 300)) return { error: `HTTP ${code}` }
-    if (contentType && !/text\/html|application\/xhtml|text\/plain|\/json|[+/]xml/i.test(contentType)) {
+    if (contentType && !isFetchableContentType(contentType)) {
       return { error: `Unsupported content-type: ${contentType.split(';')[0]}` }
     }
     return { buffer: body, contentType, finalUrl: finalUrlEff, viaProxy: proxy }

package/spa.js

@@ -4,6 +4,11 @@
 // When absent, reads fall back to static extraction with a clear hint.
 
 let browserPromise = null
+const closingBrowsers = new Set()
+
+function renderError(error) {
+  return `Render failed: ${String(error?.message || error).split('\n', 1)[0].slice(0, 500)}`
+}
 
 // Heuristic: a page whose HTML carries many <script> tags is likely a
 // client-rendered SPA (Vue/React) whose body lives only after JS execution.
@@ -33,12 +38,19 @@ async function getBrowser() {
   if (!browserPromise) {
     // A failed launch (playwright later installed / chromium download finished)
     // must not stay cached as a rejected promise — reset so the next call retries.
-    browserPromise = import('playwright')
+    const pending = import('playwright')
       .then(({ chromium }) => chromium.launch({ headless: true }))
+      .then((browser) => {
+        browser.once('disconnected', () => {
+          if (browserPromise === pending) browserPromise = null
+        })
+        return browser
+      })
       .catch((e) => {
-        browserPromise = null
+        if (browserPromise === pending) browserPromise = null
         throw e
       })
+    browserPromise = pending
   }
   return browserPromise
 }
@@ -49,12 +61,23 @@ export async function renderPage(url, externalSignal) {
   let browser
   try {
     browser = await getBrowser()
-  } catch {
-    return { error: 'SPA 渲染需 playwright（npm i playwright && npx playwright install chromium）' }
+  } catch (error) {
+    if (externalSignal && externalSignal.aborted) return { error: 'cancelled' }
+    return { error: error?.code === 'ERR_MODULE_NOT_FOUND'
+      ? 'SPA 渲染需 playwright（npm i playwright && npx playwright install chromium）'
+      : renderError(error) }
   }
+  if (externalSignal && externalSignal.aborted) return { error: 'cancelled' }
   let page
+  let pageClosing
+  const closePage = () => {
+    if (page && !pageClosing) pageClosing = page.close().catch(() => {})
+    return pageClosing
+  }
   try {
     page = await browser.newPage()
+    externalSignal?.addEventListener('abort', closePage, { once: true })
+    if (externalSignal && externalSignal.aborted) return { error: 'cancelled' }
     // 'domcontentloaded' instead of 'networkidle': heartbeat-polling sites
     // (qq-news, juejin) never go idle and would time out at 30s. Instead wait
     // for the DOM to stabilize (content stops growing) up to 10s — SPA paint
@@ -72,7 +95,8 @@ export async function renderPage(url, externalSignal) {
         .evaluate(() => (document.body ? document.body.innerHTML.length : 0))
         .catch(() => -1)
       if (Date.now() - t0 > 10000) break
-      if (len === -1 && ++evalFails >= 2) break
+      evalFails = len === -1 ? evalFails + 1 : 0
+      if (evalFails >= 2) break
       // stop once two consecutive reads agree — including empty bodies (a
       // blank page should not burn the full 10s poll)
       if (len === prevLen && prevLen >= 0) break
@@ -93,25 +117,33 @@ export async function renderPage(url, externalSignal) {
         if (!looksLikeChallenge(html)) break
       }
     }
+    if (externalSignal && externalSignal.aborted) return { error: 'cancelled' }
     const finalUrl = page.url()
     return { html, finalUrl }
   } catch (e) {
     if (externalSignal && externalSignal.aborted) return { error: 'cancelled' }
-    return { error: `Render failed: ${e.message}` }
+    return { error: renderError(e) }
   } finally {
-    if (page) await page.close().catch(() => {})
+    externalSignal?.removeEventListener('abort', closePage)
+    await closePage()
   }
 }
 
 // Called on plugin unload (temporal composability): release the browser.
 export async function closeBrowser() {
-  if (!browserPromise) return
   const p = browserPromise
   browserPromise = null
-  try {
-    const b = await p
-    await b.close().catch(() => {})
-  } catch {
-    // launch never completed — nothing to close
+  if (p) {
+    const closing = (async () => {
+      try {
+        const b = await p
+        await b.close()
+      } catch {
+        // A failed launch or an already closed browser needs no further cleanup.
+      }
+    })()
+    closingBrowsers.add(closing)
+    closing.finally(() => closingBrowsers.delete(closing))
   }
+  await Promise.all(closingBrowsers)
 }