dsh-read-url 0.4.9 → 1.1.0

package/index.js

@@ -7,15 +7,18 @@
 //     (temporal composability, docs/cordis-primer.md)
 //   - cooperative tool-call timeout via ToolDefinition.timeoutMs + exec.signal
 // Focus: token economy + smarter extraction for DSH agents.
-//   - charset auto-detect (GBK/GB2312/UTF-8/Big5) via built-in TextDecoder
+//   - charset auto-detect (BOM UTF-16 / GBK/GB2312/UTF-8/Big5/Shift-JIS)
+//   - content dispatch: HTML / JSON APIs / RSS-Atom feeds natively
 //   - clean main-content extraction (heuristic; optional @mozilla/readability upgrade path)
+//     + text-density fallback on the body path
+//   - auto-joined pagination, page metadata (published/author)
 //   - HTML -> Markdown / plain text, paragraph-aligned smart truncation
 //   - session-level cache, compact text render (lowest token cost)
 import { TextDecoder } from 'node:util'
-import { looksLikeSpa, renderPage, closeBrowser } from './spa.js'
-import { detectProxy, fetchViaCurlProxy } from './proxy-fallback.js'
+import { looksLikeSpa, renderPage, closeBrowser, looksLikeChallenge } from './spa.js'
+import { detectProxy, fetchViaCurlProxy, looksBinary } from './proxy-fallback.js'
 // Re-export for tests / programmatic (PTC) cleanup.
-export { closeBrowser, renderPage, looksLikeSpa } from './spa.js'
+export { closeBrowser, renderPage, looksLikeSpa, looksLikeChallenge } from './spa.js'
 
 export const name = 'dsh-read-url'
 export const inject = ['tools']
@@ -29,6 +32,8 @@ const DEFAULTS = {
   cacheTtlMs: 300000,
   cacheMax: 32,
   spaRender: true,
+  paginate: true,
+  paginateMax: 3,
   userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36',
 }
 
@@ -51,12 +56,16 @@ function getDecoder(enc) {
 }
 
 function sniffCharset(buffer, contentType) {
+  // BOM first — byte-level evidence beats any declared charset (a UTF-16 body
+  // with a latin1-read <meta> probe garbles the meta match anyway).
+  if (buffer[0] === 0xef && buffer[1] === 0xbb && buffer[2] === 0xbf) return 'utf-8'
+  if (buffer[0] === 0xff && buffer[1] === 0xfe) return 'utf-16le'
+  if (buffer[0] === 0xfe && buffer[1] === 0xff) return 'utf-16be'
   const m = /charset=["']?([\w-]+)/i.exec(contentType || '')
   if (m) return m[1]
   const head = buffer.subarray(0, 2048).toString('latin1').toLowerCase()
   const meta = /<meta[^>]+charset=["']?([\w-]+)/i.exec(head)
   if (meta) return meta[1]
-  if (buffer[0] === 0xef && buffer[1] === 0xbb && buffer[2] === 0xbf) return 'utf-8'
   return null
 }
 
@@ -77,10 +86,22 @@ export function decodeBuffer(buffer, contentType) {
   return { text, charset: enc }
 }
 
-// Direct-connect fetch, extracted so the race path can run it concurrently
-// with the proxy attempt and abort it. Returns { buffer, contentType, finalUrl }
-// on success or { error } — never throws (abort/timeout/network all mapped).
-async function directFetch(url, signal, cfg) {
+// Content types this plugin can consume. Beyond HTML: JSON (data APIs) and
+// XML (RSS/Atom feeds) are read natively — see readUrl's type dispatch.
+// Note "application/rss+xml": the separator before xml is '+', not '/'.
+const FETCHABLE_CT = /text\/html|application\/xhtml|text\/plain|\/json|[+/]xml/i
+
+// Retry-After header (seconds form) → ms, capped; undefined when absent/invalid.
+function retryAfterMs(res) {
+  const v = res.headers.get('retry-after')
+  if (!v) return undefined
+  const sec = Number(v)
+  if (!Number.isFinite(sec) || sec < 0) return undefined
+  return Math.min(sec * 1000, 5000)
+}
+
+// Single fetch attempt. Never throws; maps abort/timeout/network to { error }.
+async function directFetchOnce(url, signal, cfg) {
   try {
     const res = await fetch(url, {
       redirect: 'follow',
@@ -90,9 +111,19 @@ async function directFetch(url, signal, cfg) {
     // Early-return paths must cancel the body stream, or the connection is
     // held open undrained and cannot return to the keep-alive pool.
     const discardBody = () => { if (res.body) res.body.cancel().catch(() => {}) }
-    if (!res.ok) { discardBody(); return { error: `HTTP ${res.status} ${res.statusText}` } }
+    if (!res.ok) {
+      // Rate-limit / temporary-overload: honor Retry-After with one retry
+      // (capped at 5s so the cooperative timeout still bounds the call).
+      if (res.status === 429 || res.status === 503) {
+        const ra = retryAfterMs(res)
+        discardBody()
+        if (ra !== undefined) return { retryAfterMs: ra }
+      }
+      discardBody()
+      return { error: `HTTP ${res.status} ${res.statusText}` }
+    }
     const contentType = res.headers.get('content-type') || ''
-    if (contentType && !/text\/html|application\/xhtml|text\/plain/i.test(contentType)) {
+    if (contentType && !FETCHABLE_CT.test(contentType)) {
       discardBody()
       return { error: `Unsupported content-type: ${contentType.split(';')[0]}` }
     }
@@ -104,7 +135,12 @@ async function directFetch(url, signal, cfg) {
       if (size > cfg.maxBytes) { discardBody(); return { error: `Page exceeds ${cfg.maxBytes} bytes` } }
       chunks.push(chunk)
     }
-    return { buffer: Buffer.concat(chunks), contentType, finalUrl: res.url }
+    const buffer = Buffer.concat(chunks)
+    // Headerless responses: gate binary bodies out of the HTML pipeline.
+    if (!contentType && looksBinary(buffer)) {
+      return { error: 'Unsupported content-type (no header, binary body)' }
+    }
+    return { buffer, contentType, finalUrl: res.url }
   } catch (e) {
     if (e.name === 'AbortError' || e.name === 'TimeoutError') {
       return { error: `Timeout after ${cfg.timeoutMs}ms or cancelled` }
@@ -117,6 +153,19 @@ async function directFetch(url, signal, cfg) {
   }
 }
 
+// Direct-connect fetch with a single Retry-After-aware retry on 429/503.
+async function directFetch(url, signal, cfg) {
+  const first = await directFetchOnce(url, signal, cfg)
+  if (first && first.retryAfterMs !== undefined && !(signal && signal.aborted)) {
+    await new Promise((resolve) => {
+      const t = setTimeout(resolve, first.retryAfterMs)
+      if (signal) signal.addEventListener('abort', () => { clearTimeout(t); resolve() }, { once: true })
+    })
+    return directFetchOnce(url, signal, cfg)
+  }
+  return first
+}
+
 // Settle-all race that resolves with the FIRST SUCCESSFUL result (a value
 // with a buffer and no error). If every attempt fails it resolves with
 // { success:false, failures:[...] } once the slowest one has settled —
@@ -245,26 +294,136 @@ function textOnly(html) {
     .trim())
 }
 
+// ---- JSON / RSS / Atom content dispatch ----
+// A URL is not always an HTML page: data APIs answer JSON and content sources
+// answer feeds. Both are model-readable when compacted, so read_url handles
+// them natively instead of erroring with "Unsupported content-type".
+
+function xmlText(s) {
+  let t = String(s || '').replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, '$1')
+  // Feeds double-escape HTML in descriptions (&lt;a href=…&gt;查看全文&lt;/a&gt;):
+  // one strip-then-decode pass leaves literal tags in the rendered text
+  // (measured on sspai.com/feed). Iterate strip → decode until stable.
+  for (let i = 0; i < 3; i++) {
+    const prev = t
+    t = decodeTextEntities(t.replace(/<[^>]+>/g, ' '))
+    if (t === prev) break
+  }
+  return t.replace(/\s+/g, ' ').trim()
+}
+
+// RSS 2.0 <item> / Atom <entry> → compact list "title — url\n  summary".
+function parseFeed(xml, limit) {
+  const isAtom = /<feed[\s>]/i.test(xml)
+  const itemRe = isAtom ? /<entry[\s>][\s\S]*?<\/entry\s*>/gi : /<item[\s>][\s\S]*?<\/item\s*>/gi
+  const items = []
+  let m
+  while ((m = itemRe.exec(xml)) && items.length < limit) {
+    const it = m[0]
+    const title = xmlText(/<title[^>]*>([\s\S]*?)<\/title>/i.exec(it)?.[1] || '')
+    const linkTag = /<link[^>]*>([\s\S]*?)<\/link>/i.exec(it)
+    const linkHref = /<link[^>]+href=["']([^"']+)["']/i.exec(it)
+    const link = (linkTag && xmlText(linkTag[1])) || (linkHref && linkHref[1]) || ''
+    const desc = xmlText(/<(?:description|summary|content)[^>]*>([\s\S]*?)<\/(?:description|summary|content)>/i.exec(it)?.[1] || '')
+    if (!title && !link && !desc) continue
+    items.push({ title: title.slice(0, 120), url: link, summary: desc.slice(0, 200) })
+  }
+  const feedTitle = xmlText(/<title[^>]*>([\s\S]*?)<\/title>/i.exec(xml)?.[1] || '')
+  const lines = []
+  for (const it of items) {
+    lines.push(`- ${it.title}${it.url ? ` — ${it.url}` : ''}`)
+    if (it.summary) lines.push(`  ${it.summary}`)
+  }
+  return { title: feedTitle, count: items.length, items, text: lines.join('\n') }
+}
+
+// ---- Text-density fallback (body path only) ----
+// Pages without <article>/<main> degrade to the whole <body>: semantic noise
+// stripping then still leaves link-dominated blocks (related posts, category
+// sidebars, "hot articles" widgets) that carry little reading value. Split the
+// body at block-level open tags and drop segments whose text is mostly links.
+// Standard-container pages (article/main) never reach this code path.
+const BLOCK_OPEN_RE = /<(?:p|div|section|article|main|h[1-6]|table|tbody|tr|ul|ol|li|blockquote|pre|dl|figure|figcaption|fieldset)\b[^>]*>/gi
+
+function linkTextLength(html) {
+  let n = 0
+  const re = /<a[\s>][\s\S]*?<\/a\s*>/gi
+  let m
+  while ((m = re.exec(html))) n += textOnly(m[0]).length
+  return n
+}
+
+export function densityFilter(html) {
+  const marks = []
+  BLOCK_OPEN_RE.lastIndex = 0
+  let m
+  while ((m = BLOCK_OPEN_RE.exec(html))) marks.push(m.index)
+  if (marks.length < 2) return html
+  const segs = []
+  if (marks[0] > 0) segs.push(html.slice(0, marks[0]))
+  for (let k = 0; k < marks.length; k++) {
+    segs.push(html.slice(marks[k], k + 1 < marks.length ? marks[k + 1] : html.length))
+  }
+  const kept = segs.filter((seg) => {
+    const text = textOnly(seg)
+    if (text.length < 2) return false
+    // short link-dominated segment = nav item / recommendation widget entry
+    if (text.length < 300 && linkTextLength(seg) > text.length * 0.65) return false
+    return true
+  })
+  return kept.length ? kept.join('') : html
+}
+
+// Depth-counted block extraction: returns html from the opening tag at
+// openIdx through its BALANCED closing tag. A non-greedy regex stops at the
+// first nested </div> — measured: gnu.org "content" div cut to 2700 chars,
+// naver.com "container" to 115 (zero text). Container divs nest heavily.
+function tagBlockAt(html, openIdx, tagName) {
+  const re = new RegExp(`</?${tagName}(?=[\\s>])`, 'gi')
+  re.lastIndex = openIdx
+  let depth = 0
+  let m
+  while ((m = re.exec(html))) {
+    depth += m[0][1] === '/' ? -1 : 1
+    if (depth === 0) {
+      const close = html.indexOf('>', m.index)
+      return close > 0 ? html.slice(openIdx, close + 1) : null
+    }
+  }
+  return null
+}
+
 export function pickMain(html) {
   // Aggregation pages (e.g. blog homepages) have one <article> per item:
-  // collect all of them instead of only the first.
+  // collect all of them instead of only the first. A tiny <article> (e.g. a
+  // newsletter card — measured on about.gitlab.com: 71 chars of text while
+  // the real content sits in <main>) falls through to <main> when present.
   const articles = [...html.matchAll(/<article[\s>][\s\S]*?<\/article\s*>/gi)]
-  if (articles.length) return articles.map((a) => a[0]).join('\n')
-  const mainTag =
-    /<(main|div)[^>]*role=["']main["'][\s>][\s\S]*?<\/\1\s*>/i.exec(html) ||
-    // Bare <main> without role="main": doc generators (VitePress/VuePress,
-    // MDN, MDX sites) mark the body with a plain <main> tag. Falling back to
-    // the whole <body> would drag in the top/local nav (menu, outline, skip
-    // link) — the extraction quality loss we saw on vuejs.org.
-    /<main[\s>][\s\S]*?<\/main\s*>/i.exec(html)
-  if (mainTag) return mainTag[0]
+  const hasMain = /<main[\s>]/i.test(html)
+  if (articles.length && (!hasMain || textOnly(articles.map((a) => a[0]).join('')).length >= 200)) {
+    return articles.map((a) => a[0]).join('\n')
+  }
+  const roleMain = /<(main|div)[^>]*role=["']main["'][\s>]/i.exec(html)
+  if (roleMain) {
+    const block = tagBlockAt(html, roleMain.index, roleMain[1].toLowerCase())
+    if (block) return block
+  }
+  // Bare <main> without role="main": doc generators (VitePress/VuePress,
+  // MDN, MDX sites) mark the body with a plain <main> tag. Falling back to
+  // the whole <body> would drag in the top/local nav (menu, outline, skip
+  // link) — the extraction quality loss we saw on vuejs.org.
+  const bareMain = /<main[\s>]/i.exec(html)
+  if (bareMain) {
+    const block = tagBlockAt(html, bareMain.index, 'main')
+    if (block) return block
+  }
   const body = /<body[\s>]/i.exec(html)
   if (body) {
     const after = html.slice(body.index)
     const end = after.search(/<\/body>/i)
-    return end > 0 ? after.slice(0, end) : after
+    return densityFilter(end > 0 ? after.slice(0, end) : after)
   }
-  return html
+  return densityFilter(html)
 }
 
 // Some sites (e.g. baidu.com) ship CSS/HTML inside hidden <textarea> with
@@ -290,7 +449,33 @@ function escInline(s) {
   return s.replace(/([\\`*_[\]])/g, '\\$1')
 }
 
+// <img> is a void element (never matches the paired-tag regex in either md
+// walker): convert descriptive images to markdown BEFORE the tag walk, via a
+// sentinel so escInline cannot escape the generated brackets. Decorative
+// images (empty alt) are dropped — the model cannot see pixels anyway.
+const IMG_SENTINEL = '\u0001'
+function imgsToMarkdown(html) {
+  const store = []
+  const out = html.replace(/<img\b[^>]*>/gi, (tag) => {
+    const alt = /alt=["']([^"']*)["']/i.exec(tag)
+    const src = /src=["']([^"']+)["']/i.exec(tag)
+    store.push(src && alt && alt[1].trim() ? `![${alt[1].trim()}](${src[1]})` : '')
+    return `${IMG_SENTINEL}${store.length - 1}${IMG_SENTINEL}`
+  })
+  return {
+    html: out,
+    // Unknown sentinel indexes are PRESERVED (a nested walker's restore must
+    // not eat sentinels owned by an outer frame — blockMd recurses into itself).
+    restore: (s) => s.replace(new RegExp(`${IMG_SENTINEL}(\\d+)${IMG_SENTINEL}`, 'g'), (m, i) => {
+      const v = store[Number(i)]
+      return v === undefined ? m : v
+    }),
+  }
+}
+
 export function inlineMd(html) {
+  const img = imgsToMarkdown(html)
+  html = img.html
   let out = ''
   const re = /<([a-zA-Z0-9]+)((?:"[^"]*"|'[^']*'|[^'">])*)>([\s\S]*?)<\/\1>|<[^>]+>|([^<]+)/g
   let m
@@ -312,11 +497,13 @@ export function inlineMd(html) {
     else if (tag === 'br') out += '  \n'
     else out += inner
   }
-  return out.replace(/[ \t]+/g, ' ').trim()
+  return img.restore(out.replace(/[ \t]+/g, ' ').trim())
 }
 
 export function blockMd(html) {
   let out = ''
+  const img = imgsToMarkdown(html)
+  html = img.html
   const re = /<([a-zA-Z0-9]+)((?:"[^"]*"|'[^']*'|[^'">])*)>([\s\S]*?)<\/\1>|<[^>]+>|([^<]+)/g
   let m
   while ((m = re.exec(html))) {
@@ -345,7 +532,11 @@ export function blockMd(html) {
       out += `\n\n> ${t.replace(/\n/g, '\n> ')}`
     } else if (tag === 'pre') {
       const code = inner.replace(/<[^>]+>/g, '').trim()
-      out += `\n\n\`\`\`\n${code}\n\`\`\``
+      // Language hint from common highlighter conventions: the model reads
+      // ```js fenced blocks far more accurately than unlabelled ones.
+      const lang =
+        /<code[^>]+class=["'][^"']*(?:language|lang)-([\w#+.-]+)["']/i.exec(inner)?.[1] || ''
+      out += `\n\n\`\`\`${lang}\n${code}\n\`\`\``
     } else if (tag === 'code') {
       out += `\`${textOnly(inner)}\``
     } else if (tag === 'ul' || tag === 'ol') {
@@ -388,6 +579,44 @@ export function blockMd(html) {
       if (t) out += t
     }
   }
+  return img.restore(out)
+}
+
+// Page-level metadata the model actually asks about (who wrote it, when) —
+// cheap to harvest from <meta>, previously thrown away.
+function extractMeta(html) {
+  const get = (names) => {
+    for (const n of names) {
+      const m = new RegExp(`<meta[^>]+(?:property|name)=["']${n}["'][^>]+content=["']([^"']+)["']`, 'i').exec(html)
+      if (m) return decodeTextEntities(m[1]).replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim().slice(0, 80)
+    }
+    return ''
+  }
+  return {
+    published: get(['article:published_time', 'og:published_time', 'pubdate', 'publishdate', 'date', 'dc.date']),
+    author: get(['author', 'article:author', 'og:article:author']),
+    description: get(['og:description', 'description']),
+  }
+}
+
+// Meta-less pages (measured: ruanyifeng.com carries ZERO author/date meta
+// tags) still print the byline inside the body head: "作者：阮一峰 日期：
+// 2026年8月21日". Fill ONLY the missing fields, and only from the first
+// 600 chars of body text — deeper matches are content, not bylines.
+const BYLINE_AUTHOR_RE = /(?:作者|責任編輯|責任主编|editor)[：:\s]{1,3}([^\s，,。/|()（）[\]]{2,24})/
+const BYLINE_DATE_RE = /(?:发表日期|发布日期|发表时间|发布时间|日期|发表于|发布于|published(?:\s+on)?)[：:\s]{0,3}(\d{4}[-/.年]\s?\d{1,2}[-/.月]\s?\d{1,2}日?|\d{4}-\d{2}-\d{2}(?:[T ]\d{2}:\d{2})?)/i
+
+function bylineFallback(text, meta) {
+  const head = typeof text === 'string' ? text.slice(0, 600) : ''
+  const out = { published: meta.published, author: meta.author }
+  if (!out.author) {
+    const m = BYLINE_AUTHOR_RE.exec(head)
+    if (m) out.author = m[1].trim().slice(0, 40)
+  }
+  if (!out.published) {
+    const m = BYLINE_DATE_RE.exec(head)
+    if (m) out.published = m[1].replace(/\s+/g, ' ').trim().slice(0, 40)
+  }
   return out
 }
 
@@ -403,11 +632,24 @@ export function extract(html, mode) {
   } else {
     bodyText = textOnly(main).replace(/ +/g, ' ').trim()
   }
+  const meta = extractMeta(html)
+  // Empty-body pages (login walls, JS-only shells) still carry og:description
+  // — surface it as a fallback hint instead of nothing.
+  let text = bodyText
+  let usedDescription = false
+  if (!text && meta.description) {
+    text = meta.description
+    usedDescription = true
+  }
+  const byline = bylineFallback(text, meta)
   return {
     title: titleMatch ? textOnly(titleMatch[1]) || titleMatch[1].trim() : '',
     siteName: siteName ? siteName[1] : '',
     lang: langMatch ? langMatch[1] : '',
-    text: bodyText,
+    published: byline.published,
+    author: byline.author,
+    text,
+    usedDescription,
   }
 }
 
@@ -508,6 +750,53 @@ function hostOf(url) {
   }
 }
 
+// ---- Pagination: auto-follow "next page" ----
+// Long articles on novel/news/forum sites are split into pages; without this
+// the model had to notice the truncation and re-call the tool per page. The
+// recognition is deliberately conservative: rel=next (standard) or a short
+// anchor whose whole text is a next-page marker — no fuzzy guessing.
+const NEXT_TEXT_RE = /^(?:下一页|下页|后一页|下一頁|下頁|next page|next|›{1,3}|»{1,3}|>|older entries)$/i
+
+export function findNextLink(html, baseUrl) {
+  const resolve = (href) => {
+    try {
+      const u = new URL(href, baseUrl)
+      return u.protocol === 'http:' || u.protocol === 'https:' ? u.href : null
+    } catch {
+      return null
+    }
+  }
+  const rel = /<(?:link|a)\b[^>]+rel=["']next["'][^>]*>/i.exec(html || '')
+  if (rel) {
+    const href = /href=["']([^"']+)["']/i.exec(rel[0])
+    const r = href && resolve(href[1])
+    if (r) return r
+  }
+  const re = /<a\b[^>]+href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a\s*>/gi
+  let m
+  while ((m = re.exec(html || ''))) {
+    const t = textOnly(m[2])
+    if (t.length <= 16 && NEXT_TEXT_RE.test(t)) {
+      const r = resolve(m[1])
+      if (r) return r
+    }
+  }
+  return null
+}
+
+// Join two page bodies, dropping a repeated suffix/prefix overlap (many
+// paginated sites repeat the last paragraph or a heading on the next page).
+function joinPageText(a, b) {
+  const max = Math.min(300, a.length, b.length)
+  for (let n = max; n >= 30; n--) {
+    if (a.slice(a.length - n) === b.slice(0, n)) {
+      const rest = b.slice(n).trim()
+      return rest ? `${a}\n\n${rest}` : a
+    }
+  }
+  return `${a}\n\n${b}`
+}
+
 function sliceFrom(full, offset, maxChars) {
   const s = smartTruncate(full.fullText, maxChars, offset)
   const out = {
@@ -525,6 +814,10 @@ function sliceFrom(full, offset, maxChars) {
   }
   if (full.rendered) out.rendered = true
   if (full.spaHint) out.spaHint = full.spaHint
+  if (full.paginated > 1) out.paginated = full.paginated
+  if (full.feedCount) out.feedCount = full.feedCount
+  if (full.published) out.published = full.published
+  if (full.author) out.author = full.author
   if (Array.isArray(full.links)) out.links = full.links
   return out
 }
@@ -562,27 +855,70 @@ export async function readUrl(args, ctx, externalSignal, cfg = DEFAULTS) {
     }
   }
 
-  let html, finalUrl, charset
-  const viaSeam = await fetchViaWebSeam(ctx, url, externalSignal, cfg)
-  if (viaSeam) {
-    html = viaSeam.html
-    finalUrl = viaSeam.finalUrl
-    charset = 'provider-decoded'
-  } else {
-    const page = await fetchPage(url, externalSignal, cfg)
-    if (page.error) {
-      // Brief error cache: a failing URL usually stays failing for a while,
-      // so serve the same error from cache instead of re-fetching.
-      if (cache.size >= cfg.cacheMax) cache.delete(cache.keys().next().value)
-      cache.set(cacheKey, { time: Date.now(), error: page.error })
-      return { error: page.error }
+  const failWith = (error) => {
+    if (cache.size >= cfg.cacheMax) cache.delete(cache.keys().next().value)
+    cache.set(cacheKey, { time: Date.now(), error })
+    return { error }
+  }
+
+  // Non-HTML payloads (JSON APIs, RSS/Atom feeds) are read natively: fetch,
+  // dispatch by content-type / payload shape, render compact. Falls back to
+  // the HTML pipeline for everything else.
+  const dispatch = async () => {
+    const viaSeam = await fetchViaWebSeam(ctx, url, externalSignal, cfg)
+    if (viaSeam) {
+      const t = viaSeam.html.trim()
+      if (t.startsWith('{') || t.startsWith('[')) {
+        try {
+          return { kind: 'json', text: JSON.stringify(JSON.parse(t), null, 1) }
+        } catch { /* not JSON — keep HTML pipeline */ }
+      }
+      return { kind: 'html', html: viaSeam.html, finalUrl: viaSeam.finalUrl, charset: 'provider-decoded' }
     }
+    const page = await fetchPage(url, externalSignal, cfg)
+    if (page.error) return { error: page.error }
+    const ct = (page.contentType || '').split(';')[0].toLowerCase().trim()
     const decoded = decodeBuffer(page.buffer, page.contentType)
-    html = decoded.text
-    finalUrl = page.finalUrl || url
-    charset = decoded.charset
+    if (/json$/.test(ct)) {
+      try {
+        return { kind: 'json', text: JSON.stringify(JSON.parse(decoded.text), null, 1) }
+      } catch { /* invalid JSON — serve through the HTML pipeline as text */ }
+    }
+    if (/[+/]xml$/.test(ct)) {
+      if (/<urlset[\s>]/i.test(decoded.text) || /<sitemapindex[\s>]/i.test(decoded.text)) {
+        return { error: 'Unsupported content (XML sitemap)' }
+      }
+      if (/<rss[\s>]/i.test(decoded.text) || /<feed[\s>]/i.test(decoded.text)) {
+        return { kind: 'feed', feed: parseFeed(decoded.text, cfg.maxLinks) }
+      }
+    }
+    return { kind: 'html', html: decoded.text, finalUrl: page.finalUrl || url, charset: decoded.charset }
+  }
+  const payload = await dispatch()
+  if (payload.error) return failWith(payload.error)
+
+  if (payload.kind === 'json') {
+    const full = { url, title: '', siteName: hostOf(url), lang: '', charset: 'json', mode: 'json', fullText: payload.text, paginated: 1 }
+    if (cache.size >= cfg.cacheMax) cache.delete(cache.keys().next().value)
+    cache.set(cacheKey, { time: Date.now(), full })
+    return sliceFrom(full, offset, maxChars)
+  }
+  if (payload.kind === 'feed') {
+    const f = payload.feed
+    const full = {
+      url, title: f.title || '', siteName: hostOf(url), lang: '', charset: 'feed',
+      mode: 'feed', fullText: f.text, paginated: 1, feedCount: f.count,
+      links: args.includeLinks === true ? f.items : undefined,
+    }
+    if (cache.size >= cfg.cacheMax) cache.delete(cache.keys().next().value)
+    cache.set(cacheKey, { time: Date.now(), full })
+    return sliceFrom(full, offset, maxChars)
   }
 
+  let html = payload.html
+  let finalUrl = payload.finalUrl || url
+  const charset = payload.charset
+
   let extracted = extract(html, mode)
   // Optional readability upgrade: cleaner article extraction when installed.
   const ext = await getReadabilityExtractor()
@@ -604,24 +940,78 @@ export async function readUrl(args, ctx, externalSignal, cfg = DEFAULTS) {
 
   // Optional SPA enhancement: if the page looks client-rendered and static
   // extraction found almost nothing, try headless rendering (playwright).
+  // Beyond script-heavy shells (looksLikeSpa, ≥5 <script> tags), a page whose
+  // extraction is EMPTY but carries any <script> may be a JS-redirect shell
+  // (measured: taptap.cn — 2 scripts, empty <body>, redirects via JS) — with
+  // no text to lose, rendering is always worth one bounded attempt. Pages
+  // with no scripts at all cannot change under rendering and are skipped.
   let rendered = false
   let spaHint = ''
   let renderedHtml = null
-  if (cfg.spaRender !== false && looksLikeSpa(html) && (!extracted.text || extracted.text.length < 200)) {
+  const needsRender = !extracted.text || extracted.text.length < 200
+  const scriptShell = !extracted.text && /<script[\s>]/i.test(html)
+  if (cfg.spaRender !== false && needsRender && (looksLikeSpa(html) || scriptShell)) {
     const rr = await renderPage(finalUrl || url, externalSignal)
     if (rr.html) {
       renderedHtml = rr.html
-      const r2 = upgrade(extract(rr.html, mode), rr.html)
-      if (r2.text && r2.text.length > extracted.text.length + 50) {
-        extracted = r2
-        finalUrl = rr.finalUrl || finalUrl
-        rendered = true
+      // A challenge interstitial (Cloudflare "Just a moment...") must never
+      // be accepted as an improvement — it can carry MORE text than the real
+      // static body (measured: 259 chars of verification prose vs 135 chars
+      // of real content on ruanyifeng.com) and would evict the latter.
+      if (looksLikeChallenge(rr.html)) {
+        renderedHtml = null // pagination/links must not read the interstitial DOM
+        spaHint = extracted.text
+          ? '已渲染但被人机验证页拦截，保留静态提取结果'
+          : '已渲染但被人机验证页拦截（无静态内容可用）'
+      } else {
+        const r2 = upgrade(extract(rr.html, mode), rr.html)
+        // Accept the rendered text when it meaningfully beats the static one.
+        // From an EMPTY static extraction even a short body is a real gain
+        // (measured: taptap-style JS shells render 30-60 chars of real content
+        // that the fixed +50 margin would reject).
+        const gain = r2.text.length - extracted.text.length
+        const accept = r2.text && (gain > 50 || (!extracted.text && r2.text.length >= 20))
+        if (accept) {
+          extracted = r2
+          finalUrl = rr.finalUrl || finalUrl
+          rendered = true
+        } else if (!extracted.text) {
+          spaHint = '已渲染仍无可读内容（可能登录墙或反爬拦截）'
+        }
       }
     } else {
       spaHint = rr.error
     }
   }
 
+  // Auto-pagination: follow the "next page" chain (bounded, same-host, no
+  // loops). Continuation pages go through the fast static path — a paginated
+  // SPA chain would cost one full render per page, out of proportion.
+  let paginated = 1
+  const paginateMax = cfg.paginate === false ? 1 : Math.max(1, Math.min(10, Number(cfg.paginateMax) || 3))
+  const startHost = hostOf(finalUrl || url)
+  const seen = new Set([normalizeUrl(finalUrl || url)])
+  let nextUrl = paginateMax > 1 ? findNextLink(renderedHtml || html, finalUrl || url) : null
+  while (
+    nextUrl && paginated < paginateMax &&
+    sameHost(nextUrl, startHost) &&
+    !(externalSignal && externalSignal.aborted)
+  ) {
+    const nu = normalizeUrl(nextUrl)
+    if (!nu || seen.has(nu)) break
+    seen.add(nu)
+    const page = await fetchPage(nextUrl, externalSignal, cfg)
+    if (page.error) break
+    const ct = (page.contentType || '').split(';')[0].toLowerCase().trim()
+    if (!/html|xhtml/.test(ct)) break
+    const nextHtml = decodeBuffer(page.buffer, page.contentType).text
+    const nextEx = extract(nextHtml, mode)
+    if (!nextEx.text || nextEx.text.length < 20) break
+    extracted.text = joinPageText(extracted.text, nextEx.text)
+    paginated++
+    nextUrl = findNextLink(nextHtml, page.finalUrl || nextUrl)
+  }
+
   const full = {
     url: finalUrl,
     title: extracted.title || '',
@@ -632,6 +1022,9 @@ export async function readUrl(args, ctx, externalSignal, cfg = DEFAULTS) {
     fullText: extracted.text,
     rendered,
     spaHint,
+    paginated,
+    published: extracted.published || '',
+    author: extracted.author || '',
     // Links come from the rendered DOM when available — otherwise the SPA
     // chain (content -> links -> next page) would break for JS-only pages.
     links: args.includeLinks === true ? extractLinks(renderedHtml || html, cfg.maxLinks, finalUrl) : undefined,
@@ -652,7 +1045,11 @@ function renderResult(value) {
   const meta = []
   if (r.siteName && r.siteName !== host) meta.push(r.siteName)
   if (r.lang) meta.push(r.lang)
-  if (r.charset) meta.push(`charset ${r.charset}`)
+  if (r.charset && r.charset !== 'json' && r.charset !== 'feed') meta.push(`charset ${r.charset}`)
+  if (r.published) meta.push(r.published)
+  if (r.author) meta.push(`by ${r.author}`)
+  if (r.feedCount) meta.push(`feed · ${r.feedCount} 条`)
+  if (r.mode === 'json') meta.push('json')
   if (meta.length) lines.push(meta.join(' · '))
   if (r.charsStart > 0) {
     lines.push(`(chars ${r.charsStart}+${r.charsReturned}/${r.charsTotal} — offset 续读)`)
@@ -662,6 +1059,7 @@ function renderResult(value) {
     lines.push(`(chars ${r.charsTotal})`)
   }
   if (r.cached) lines.push('(cached)')
+  if (r.paginated > 1) lines.push(`(已自动拼接 ${r.paginated} 页)`)
   if (r.rendered) lines.push('(rendered — JS 执行后内容)')
   if (!r.text) {
     if (r.charsStart > 0 && !r.truncated) {
@@ -707,6 +1105,7 @@ function readLinksTool(ctx, cfg) {
     },
     // Covers SPA-render fallback: full fetch timeout + playwright render cap.
     timeoutMs: Math.max(45000, cfg.timeoutMs + 45000),
+    isConcurrencySafe: () => true,
     output: {
       schema: {
         type: 'object',
@@ -737,10 +1136,14 @@ function readLinksTool(ctx, cfg) {
       }
       let links = extractLinks(html, limit, finalUrl)
       // SPA fallback: a JS-only page yields no links from its static HTML;
-      // render it and re-extract when the static result looks empty.
-      if (links.length < 3 && cfg.spaRender !== false && looksLikeSpa(html)) {
+      // render it and re-extract when the static result looks empty. Any
+      // <script> counts as renderable when there are no links at all — the
+      // static shell may be a JS-redirect (same rationale as read_url).
+      if (links.length < 3 && cfg.spaRender !== false && (looksLikeSpa(html) || (links.length === 0 && /<script[\s>]/i.test(html)))) {
         const rr = await renderPage(finalUrl || url, exec && exec.signal)
-        if (rr.html) {
+        // A challenge interstitial's links are the challenge's own scripts —
+        // never an improvement over whatever the static HTML offered.
+        if (rr.html && !looksLikeChallenge(rr.html)) {
           const rl = extractLinks(rr.html, limit, rr.finalUrl || finalUrl)
           if (rl.length > links.length) {
             links = rl
@@ -805,8 +1208,11 @@ function readUrlBatchTool(ctx, cfg) {
       },
       required: ['urls'],
     },
-    // Worst case: ceil(10/4)=3 concurrency waves, each (fetch + SPA render).
+    // Worst case: ceil(10/4)=3 concurrency waves, each (fetch + SPA render).
     timeoutMs: Math.max(60000, cfg.timeoutMs * 3 + 135000),
+    // Concurrent batch calls multiply fetch pressure but stay isolated (each
+    // call owns its result array; the session cache is commutative).
+    isConcurrencySafe: () => true,
     output: {
       schema: {
         type: 'object',
@@ -974,9 +1380,13 @@ function readUrlSiteTool(ctx, cfg) {
       },
       required: ['url'],
     },
-    // Worst case: ceil(maxPages/2)=25 waves of paired fetches (no SPA render
-    // here); the cooperative signal stops the loop early once the budget is hit.
+    // Worst case: ceil(maxPages/2)=25 waves of paired fetches (no SPA render
+    // here); the cooperative signal stops the loop early once the budget is hit.
     timeoutMs: Math.max(120000, cfg.timeoutMs * 25),
+    // Crawl state (visited set, queue) is call-local; only the shared session
+    // cache is global and commutative — parallel crawls of different hosts
+    // are safe and useful (one task mapping several sites).
+    isConcurrencySafe: () => true,
     output: {
       schema: {
         type: 'object',
@@ -1035,8 +1445,10 @@ export function apply(ctx, config) {
   cfg.maxLinks = clamp(cfg.maxLinks, DEFAULTS.maxLinks, 1, 50)
   cfg.cacheTtlMs = clamp(cfg.cacheTtlMs, DEFAULTS.cacheTtlMs, 0, 3600000)
   cfg.cacheMax = clamp(cfg.cacheMax, DEFAULTS.cacheMax, 1, 256)
+  cfg.paginateMax = clamp(cfg.paginateMax, DEFAULTS.paginateMax, 1, 10)
   if (typeof cfg.userAgent !== 'string' || !cfg.userAgent.trim()) cfg.userAgent = DEFAULTS.userAgent
   if (typeof cfg.spaRender !== 'boolean') cfg.spaRender = DEFAULTS.spaRender
+  if (typeof cfg.paginate !== 'boolean') cfg.paginate = DEFAULTS.paginate
 
   // Temporal composability: unload must fully revert side effects.
   ctx.effect(() => () => {
@@ -1065,10 +1477,16 @@ export function apply(ctx, config) {
       },
       required: ['url'],
     },
-    // Pipeline budget must cover the real worst case: full fetch timeout
-    // (cfg.timeoutMs) + SPA render (playwright goto 30s cap + DOM-stability
-    // poll 10s) + margin. Derived from load-time cfg — scale with it, never clamp it.
-    timeoutMs: Math.max(45000, cfg.timeoutMs + 45000),
+    // Pipeline budget must cover the real worst case: first page fetch +
+    // SPA render (goto 30s cap + DOM-stability poll 10s) + up to paginateMax
+    // continuation fetches (static, no render). Derived from load-time cfg —
+    // scale with it, never clamp it.
+    timeoutMs: Math.max(45000, cfg.timeoutMs * (cfg.paginate === false ? 1 : cfg.paginateMax + 1) + 45000),
+    // All shared state is commutative-safe under concurrent calls (session
+    // cache Map get/set, decoder cache, one shared browser with isolated
+    // pages per render), so the agent may fan out several read_url calls in
+    // one parallel group — a real win for multi-source reading tasks.
+    isConcurrencySafe: () => true,
     output: {
       schema: {
         type: 'object',