dsh-read-url 1.1.0 → 1.3.1

package/index.js

@@ -37,9 +37,19 @@ const DEFAULTS = {
   userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36',
 }
 
-// Session-level cache: url -> { time, full } on success, { time, error } on
-// failure (short TTL so a broken URL is not re-fetched in a loop).
+// Session-level cache: url -> { time, full }. Failures live in a separate
+// small map so short-lived error entries never evict hot success pages.
 const cache = new Map()
+const failCache = new Map()
+const FAIL_CACHE_MAX = 8
+function cacheFail(key, error) {
+  if (failCache.size >= FAIL_CACHE_MAX) failCache.delete(failCache.keys().next().value)
+  failCache.set(key, { time: Date.now(), error })
+}
+function cacheStore(key, full, cacheMax) {
+  if (cache.size >= cacheMax) cache.delete(cache.keys().next().value)
+  cache.set(key, { time: Date.now(), full })
+}
 const decoders = new Map()
 
 function getDecoder(enc) {
@@ -64,8 +74,12 @@ function sniffCharset(buffer, contentType) {
   const m = /charset=["']?([\w-]+)/i.exec(contentType || '')
   if (m) return m[1]
   const head = buffer.subarray(0, 2048).toString('latin1').toLowerCase()
-  const meta = /<meta[^>]+charset=["']?([\w-]+)/i.exec(head)
+  const meta = /<meta[^>]{0,1000}charset=["']?([\w-]+)/i.exec(head)
   if (meta) return meta[1]
+  // Legacy form: <meta http-equiv="Content-Type" content="text/html; charset=gb2312">
+  // (no charset= attribute anywhere) — common on old GBK pages with no header charset.
+  const metaEquiv = /<meta[^>]{0,1000}(?:http-equiv)=["']content-type["'][^>]{0,1000}content=["'][^"']*charset=([\w-]+)/i.exec(head)
+  if (metaEquiv) return metaEquiv[1]
   return null
 }
 
@@ -207,6 +221,15 @@ async function raceFetch(url, externalSignal, cfg, proxy) {
   const winner = await raceFirstSuccess([direct, proxied])
   directCtrl.abort()
   proxyCtrl.abort()
+  if (!winner.success) {
+    // Each failure is tagged with its channel so callers read errors by
+    // label ('direct' | 'proxy') rather than by array position.
+    const labels = ['direct', 'proxy']
+    winner.failures = winner.failures.map((f, i) => ({
+      label: labels[i],
+      error: f && f.error ? String(f.error) : 'unknown error',
+    }))
+  }
   return winner
 }
 
@@ -215,10 +238,10 @@ async function fetchPage(url, externalSignal, cfg) {
   if (proxy) {
     const winner = await raceFetch(url, externalSignal, cfg, proxy)
     if (winner.success) return winner.value
-    const dErr = winner.failures[0] && winner.failures[0].error
-    const pErr = winner.failures[1]
-    const px = pErr && pErr.error ? `，代理返回 ${pErr.error}` : '，代理未连通'
-    return { error: `Fetch failed: ${dErr}（直连失败；已尝试代理 ${proxy}${px}）` }
+    const dErr = winner.failures.find((f) => f.label === 'direct')
+    const pErr = winner.failures.find((f) => f.label === 'proxy')
+    const px = pErr ? `，代理返回 ${pErr.error}` : '，代理未连通'
+    return { error: `Fetch failed: ${dErr ? dErr.error : 'unknown error'}（直连失败；已尝试代理 ${proxy}${px}）` }
   }
   // No proxy configured: plain direct connect (the original behaviour).
   const timeoutSignal = AbortSignal.timeout(cfg.timeoutMs)
@@ -266,30 +289,54 @@ const ENTITIES = {
   permil: '‰', euro: '€', pound: '£', yen: '¥', cent: '¢',
   sup2: '²', sup3: '³', frac12: '½', frac14: '¼', frac34: '¾',
 }
+// &#0; / &#127; / &#x1F; decode to control characters that pollute the model
+// context (NUL, DEL, C1 controls). Map non-whitespace controls to a space;
+// keep \t\n\r — pages legitimately use &#10; for line breaks, and flattening
+// those to spaces would destroy paragraph structure in markdown mode. Lone
+// surrogates (U+D800–DFFF) decode to replacement chars instead of leaking a
+// broken code unit into the rendered text.
+function fromSafeCodePoint(c) {
+  if (c < 0x20) return (c === 9 || c === 10 || c === 13) ? String.fromCodePoint(c) : ' '
+  if (c >= 0x7f && c <= 0x9f) return ' '
+  if (c >= 0xd800 && c <= 0xdfff) return '\uFFFD'
+  return String.fromCodePoint(c)
+}
+
 export function decodeTextEntities(text) {
   if (!text.includes('&')) return text
   return text.replace(/&(?:#(\d+)|#x([0-9a-fA-F]+)|([a-z][a-z0-9]{1,7}));?/gi, (m, dec, hex, name) => {
     if (dec !== undefined) {
       const c = Number(dec)
-      return c > 0x10ffff ? m : String.fromCodePoint(c)
+      return c > 0x10ffff ? m : fromSafeCodePoint(c)
     }
     if (hex !== undefined) {
       const c = Number.parseInt(hex, 16)
-      return c > 0x10ffff ? m : String.fromCodePoint(c)
+      return c > 0x10ffff ? m : fromSafeCodePoint(c)
     }
     const n = (name || '').toLowerCase()
     return Object.prototype.hasOwnProperty.call(ENTITIES, n) ? ENTITIES[n] : m
   })
 }
 
+// A '<' with no '>' after it can never start a real tag, but tag-scanning
+// regexes still probe the rest of the string at every such position, degrading
+// to quadratic time on pathological input. Neutralize stray '<' in the no-'>'
+// tail: text content is preserved, impossible tag openings disappear, and
+// scanning stays linear. Short tails ("a < b" in prose) are left untouched.
+function defuseLt(html) {
+  const from = html.lastIndexOf('>') + 1
+  if (html.length - from < 512 || !html.includes('<', from)) return html
+  return html.slice(0, from) + html.slice(from).replace(/</g, ' ')
+}
+
 function textOnly(html) {
-  return decodeTextEntities(html
+  return decodeTextEntities(defuseLt(html)
     .replace(/<!--[\s\S]*?-->/g, ' ')
-    .replace(/<script[\s\S]*?<\/script>/gi, ' ')
-    .replace(/<style[\s\S]*?<\/style>/gi, ' ')
-    .replace(/<textarea[\s\S]*?<\/textarea\s*>/gi, ' ')
-    .replace(/<noscript[\s\S]*?<\/noscript>/gi, ' ')
-    .replace(/<[^>]+>/g, ' ')
+    .replace(/<script[\s>][\s\S]*?<\/script>/gi, ' ')
+    .replace(/<style[\s>][\s\S]*?<\/style>/gi, ' ')
+    .replace(/<textarea[\s>][\s\S]*?<\/textarea\s*>/gi, ' ')
+    .replace(/<noscript[\s>][\s\S]*?<\/noscript>/gi, ' ')
+    .replace(/<[^>]{1,1000}>/g, ' ')
     .replace(/[ \t\r\n]+/g, ' ')
     .trim())
 }
@@ -302,11 +349,11 @@ function textOnly(html) {
 function xmlText(s) {
   let t = String(s || '').replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, '$1')
   // Feeds double-escape HTML in descriptions (&lt;a href=…&gt;查看全文&lt;/a&gt;):
-  // one strip-then-decode pass leaves literal tags in the rendered text
-  // (measured on sspai.com/feed). Iterate strip → decode until stable.
+  // one strip-then-decode pass leaves literal tags in the rendered text.
+  // Iterate strip → decode until stable.
   for (let i = 0; i < 3; i++) {
     const prev = t
-    t = decodeTextEntities(t.replace(/<[^>]+>/g, ' '))
+    t = decodeTextEntities(t.replace(/<[^>]{1,1000}>/g, ' '))
     if (t === prev) break
   }
   return t.replace(/\s+/g, ' ').trim()
@@ -314,11 +361,14 @@ function xmlText(s) {
 
 // RSS 2.0 <item> / Atom <entry> → compact list "title — url\n  summary".
 function parseFeed(xml, limit) {
+  xml = defuseLt(xml) // keeps the item scan linear
   const isAtom = /<feed[\s>]/i.test(xml)
   const itemRe = isAtom ? /<entry[\s>][\s\S]*?<\/entry\s*>/gi : /<item[\s>][\s\S]*?<\/item\s*>/gi
+  // Without a single closing tag nothing can pair, so the scan is skipped.
+  const closable = isAtom ? /<\/entry\s*>/i.test(xml) : /<\/item\s*>/i.test(xml)
   const items = []
   let m
-  while ((m = itemRe.exec(xml)) && items.length < limit) {
+  while (closable && (m = itemRe.exec(xml)) && items.length < limit) {
     const it = m[0]
     const title = xmlText(/<title[^>]*>([\s\S]*?)<\/title>/i.exec(it)?.[1] || '')
     const linkTag = /<link[^>]*>([\s\S]*?)<\/link>/i.exec(it)
@@ -343,9 +393,10 @@ function parseFeed(xml, limit) {
 // sidebars, "hot articles" widgets) that carry little reading value. Split the
 // body at block-level open tags and drop segments whose text is mostly links.
 // Standard-container pages (article/main) never reach this code path.
-const BLOCK_OPEN_RE = /<(?:p|div|section|article|main|h[1-6]|table|tbody|tr|ul|ol|li|blockquote|pre|dl|figure|figcaption|fieldset)\b[^>]*>/gi
+const BLOCK_OPEN_RE = /<(?:p|div|section|article|main|h[1-6]|table|tbody|tr|ul|ol|li|blockquote|pre|dl|figure|figcaption|fieldset)\b[^>]{0,1000}>/gi
 
 function linkTextLength(html) {
+  if (!/<\/a/i.test(html)) return 0 // no anchor can pair — skip the scan
   let n = 0
   const re = /<a[\s>][\s\S]*?<\/a\s*>/gi
   let m
@@ -375,9 +426,8 @@ export function densityFilter(html) {
 }
 
 // Depth-counted block extraction: returns html from the opening tag at
-// openIdx through its BALANCED closing tag. A non-greedy regex stops at the
-// first nested </div> — measured: gnu.org "content" div cut to 2700 chars,
-// naver.com "container" to 115 (zero text). Container divs nest heavily.
+// openIdx through its BALANCED closing tag. A non-greedy regex would stop at
+// the first nested </div>, truncating heavily nested container divs.
 function tagBlockAt(html, openIdx, tagName) {
   const re = new RegExp(`</?${tagName}(?=[\\s>])`, 'gi')
   re.lastIndex = openIdx
@@ -396,14 +446,16 @@ function tagBlockAt(html, openIdx, tagName) {
 export function pickMain(html) {
   // Aggregation pages (e.g. blog homepages) have one <article> per item:
   // collect all of them instead of only the first. A tiny <article> (e.g. a
-  // newsletter card — measured on about.gitlab.com: 71 chars of text while
-  // the real content sits in <main>) falls through to <main> when present.
-  const articles = [...html.matchAll(/<article[\s>][\s\S]*?<\/article\s*>/gi)]
+  // newsletter card while the real content sits in <main>) falls through to
+  // <main> when present.
+  const articles = /<\/article/i.test(html)
+    ? [...html.matchAll(/<article[\s>][\s\S]*?<\/article\s*>/gi)]
+    : [] // without a closer in the document nothing can pair
   const hasMain = /<main[\s>]/i.test(html)
   if (articles.length && (!hasMain || textOnly(articles.map((a) => a[0]).join('')).length >= 200)) {
     return articles.map((a) => a[0]).join('\n')
   }
-  const roleMain = /<(main|div)[^>]*role=["']main["'][\s>]/i.exec(html)
+  const roleMain = /<(main|div)[^>]{0,1000}role=["']main["'][\s>]/i.exec(html)
   if (roleMain) {
     const block = tagBlockAt(html, roleMain.index, roleMain[1].toLowerCase())
     if (block) return block
@@ -427,21 +479,40 @@ export function pickMain(html) {
 }
 
 // Some sites (e.g. baidu.com) ship CSS/HTML inside hidden <textarea> with
-// entity-escaped tags (&lt;style&gt;...). Reveal them so noise rules can strip
-// them; then run noise removal again.
+// entity-escaped tags (&lt;style&gt;...). Reveal TAG-SHAPED sequences so noise
+// rules can strip them. &lt; is revealed only when a tag name follows: a bare
+// &lt; from prose ("a &lt; b") would be swallowed by the tag stripper up to
+// the next real '>'. Bare &gt; / &quot; never open a strip span and are
+// revealed unconditionally.
 function revealEscapedTags(html) {
   return html
-    .replace(/&lt;/gi, '<')
+    .replace(/&lt;(?=\/?[a-zA-Z!])/gi, '<')
     .replace(/&gt;/gi, '>')
     .replace(/&quot;/gi, '"')
 }
 
 function stripNoise(mainHtml) {
-  return mainHtml
-    .replace(/<!--[\s\S]*?-->/g, ' ')
-    .replace(/<(textarea|style|script|noscript|template)[\s>][\s\S]*?<\/\1\s*>/gi, ' ')
-    .replace(/<(nav|footer|header|aside|form|iframe|svg|canvas|dialog)[\s>][\s\S]*?<\/\1>/gi, ' ')
-    .replace(/<([a-z][a-z0-9]*)[^>]+class=["'][^"']*(ad-|ads|advert|banner|sidebar|social|share|comment|popup|modal|cookie)[^"']*["'][^>]*>[\s\S]*?<\/\1>/gi, ' ')
+  // Attribute runs are bounded so a '<' whose '>' is far away costs a bounded
+  // scan instead of a run to that '>'; real attributes never approach 1000
+  // chars. Container groups only include tags whose closing tag exists in the
+  // input — absent closers cannot pair, so scanning for their body is wasted
+  // work.
+  const closers = new Set()
+  for (const c of mainHtml.matchAll(/<\/([a-z0-9]+)/gi)) closers.add(c[1].toLowerCase())
+  const group = (names) => names.filter((n) => closers.has(n))
+  const raw = group(['textarea', 'style', 'script', 'noscript', 'template'])
+  const box = group(['nav', 'footer', 'header', 'aside', 'form', 'iframe', 'svg', 'canvas', 'dialog'])
+  let out = mainHtml.replace(/<!--[\s\S]*?-->/g, ' ')
+  if (raw.length) {
+    out = out.replace(new RegExp(`<(${raw.join('|')})[\\s>][\\s\\S]*?<\\/\\1\\s*>`, 'gi'), ' ')
+  }
+  if (box.length) {
+    out = out.replace(new RegExp(`<(${box.join('|')})[\\s>][\\s\\S]*?<\\/\\1>`, 'gi'), ' ')
+  }
+  return out.replace(
+    /<([a-z][a-z0-9]*)[^>]{1,1000}class=["'][^"']{0,300}(ad-|ads|advert|banner|sidebar|social|share|comment|popup|modal|cookie)[^"']{0,300}["'][^>]{0,1000}>[\s\S]*?<\/\1>/gi,
+    ' ',
+  )
 }
 
 // ---- HTML -> Markdown (lightweight, tag-state machine) ----
@@ -456,7 +527,7 @@ function escInline(s) {
 const IMG_SENTINEL = '\u0001'
 function imgsToMarkdown(html) {
   const store = []
-  const out = html.replace(/<img\b[^>]*>/gi, (tag) => {
+  const out = html.replace(/<img\b[^>]{0,1000}>/gi, (tag) => {
     const alt = /alt=["']([^"']*)["']/i.exec(tag)
     const src = /src=["']([^"']+)["']/i.exec(tag)
     store.push(src && alt && alt[1].trim() ? `![${alt[1].trim()}](${src[1]})` : '')
@@ -473,11 +544,29 @@ function imgsToMarkdown(html) {
   }
 }
 
-export function inlineMd(html) {
+// An open tag whose closing tag appears nowhere in the input can never pair;
+// removing such openers up front keeps the walker from rescanning the rest of
+// the string at every position, so the walk stays linear. The walkers drop
+// these tags either way, so the rendered output is unchanged. The depth cap
+// bounds recursion — real pages never nest hundreds of levels; beyond the cap
+// the walker degrades to plain text.
+const OPEN_TAG_RE = /<([a-zA-Z0-9]+)((?:"[^"]*"|'[^']*'|[^'">]){0,1000})>/g
+const MD_MAX_DEPTH = 100
+function stripUnmatchedOpeners(html) {
+  const closers = new Set()
+  for (const m of html.matchAll(/<\/([a-zA-Z0-9]+)/g)) closers.add(m[1].toLowerCase())
+  html = defuseLt(html)
+  return html.replace(OPEN_TAG_RE, (m, name) => (closers.has(name.toLowerCase()) ? m : ''))
+}
+
+export function inlineMd(html, depth = 0) {
+  if (depth > MD_MAX_DEPTH) {
+    return escInline(decodeTextEntities(html.replace(/<[^>]{1,1000}>/g, ' ')).replace(/\s+/g, ' ').trim())
+  }
   const img = imgsToMarkdown(html)
-  html = img.html
+  html = stripUnmatchedOpeners(img.html)
   let out = ''
-  const re = /<([a-zA-Z0-9]+)((?:"[^"]*"|'[^']*'|[^'">])*)>([\s\S]*?)<\/\1>|<[^>]+>|([^<]+)/g
+  const re = /<([a-zA-Z0-9]+)((?:"[^"]*"|'[^']*'|[^'">]){0,1000})>([\s\S]*?)<\/\1>|<[^>]{1,1000}>|([^<]+)/g
   let m
   while ((m = re.exec(html))) {
     if (m[4] !== undefined) {
@@ -487,10 +576,13 @@ export function inlineMd(html) {
     if (!m[1]) continue
     const tag = m[1].toLowerCase()
     if (tag === 'style' || tag === 'script' || tag === 'textarea' || tag === 'template' || tag === 'noscript') continue
-    const inner = inlineMd(m[3])
+    const inner = inlineMd(m[3], depth + 1)
     if (tag === 'a') {
       const href = /href=["']([^"']+)["']/i.exec(m[2])
-      out += href ? `[${inner}](${href[1]})` : inner
+      // A raw ')' in the target would terminate the markdown link early
+      // (common on wiki/framework URLs) — percent-encode parens to keep the
+      // link parseable by the model.
+      out += href ? `[${inner}](${href[1].replace(/\)/g, '%29').replace(/\(/g, '%28')})` : inner
     } else if (tag === 'strong' || tag === 'b') out += `**${inner}**`
     else if (tag === 'em' || tag === 'i') out += `*${inner}*`
     else if (tag === 'code') out += `\`${inner}\``
@@ -500,11 +592,14 @@ export function inlineMd(html) {
   return img.restore(out.replace(/[ \t]+/g, ' ').trim())
 }
 
-export function blockMd(html) {
-  let out = ''
+export function blockMd(html, depth = 0) {
+  if (depth > MD_MAX_DEPTH) {
+    return decodeTextEntities(html.replace(/<[^>]{1,1000}>/g, ' ')).replace(/\s+/g, ' ').trim()
+  }
   const img = imgsToMarkdown(html)
-  html = img.html
-  const re = /<([a-zA-Z0-9]+)((?:"[^"]*"|'[^']*'|[^'">])*)>([\s\S]*?)<\/\1>|<[^>]+>|([^<]+)/g
+  html = stripUnmatchedOpeners(img.html)
+  let out = ''
+  const re = /<([a-zA-Z0-9]+)((?:"[^"]*"|'[^']*'|[^'">]){0,1000})>([\s\S]*?)<\/\1>|<[^>]{1,1000}>|([^<]+)/g
   let m
   while ((m = re.exec(html))) {
     if (m[4] !== undefined) {
@@ -520,18 +615,21 @@ export function blockMd(html) {
     }
     if (tag === 'h1' || tag === 'h2' || tag === 'h3' || tag === 'h4' || tag === 'h5' || tag === 'h6') {
       const level = '#'.repeat(Number(tag[1]))
-      out += `\n\n${level} ${inlineMd(inner)}`
+      out += `\n\n${level} ${inlineMd(inner, depth + 1)}`
     } else if (tag === 'p') {
-      const t = inlineMd(inner)
+      const t = inlineMd(inner, depth + 1)
       if (t) out += `\n\n${t}`
     } else if (tag === 'div' || tag === 'section' || tag === 'article' || tag === 'main') {
-      const t = blockMd(inner)
+      const t = blockMd(inner, depth + 1)
       if (t) out += `\n\n${t}`
     } else if (tag === 'blockquote') {
-      const t = blockMd(inner).trim()
+      const t = blockMd(inner, depth + 1).trim()
       out += `\n\n> ${t.replace(/\n/g, '\n> ')}`
     } else if (tag === 'pre') {
-      const code = inner.replace(/<[^>]+>/g, '').trim()
+      // Strip wrapper tags FIRST, then decode entities — a literal &lt;div&gt;
+      // in the code must become <div> in the fenced block, but a real nested
+      // tag must not survive the strip to be rendered as markdown.
+      const code = decodeTextEntities(inner.replace(/<[^>]{1,1000}>/g, '')).trim()
       // Language hint from common highlighter conventions: the model reads
       // ```js fenced blocks far more accurately than unlabelled ones.
       const lang =
@@ -540,29 +638,38 @@ export function blockMd(html) {
     } else if (tag === 'code') {
       out += `\`${textOnly(inner)}\``
     } else if (tag === 'ul' || tag === 'ol') {
-      const liRe = /<li[^>]*>([\s\S]*?)<\/li>/gi
-      let lm, idx = 1, items = []
-      while ((lm = liRe.exec(inner))) {
-        const t = inlineMd(lm[1])
-        items.push(tag === 'ol' ? `${idx}. ${t}` : `- ${t}`)
-        idx++
+      // Without a single </li> nothing can pair, so the item scan is skipped.
+      const items = []
+      if (inner.includes('</li')) {
+        const liRe = /<li[^>]{0,1000}>([\s\S]*?)<\/li>/gi
+        let lm, idx = 1
+        while ((lm = liRe.exec(inner))) {
+          const t = inlineMd(lm[1], depth + 1)
+          items.push(tag === 'ol' ? `${idx}. ${t}` : `- ${t}`)
+          idx++
+        }
       }
       if (items.length) out += `\n\n${items.join('\n')}`
     } else if (tag === 'li') {
-      out += `\n- ${inlineMd(inner)}`
+      out += `\n- ${inlineMd(inner, depth + 1)}`
     } else if (tag === 'br') {
       out += '  \n'
     } else if (tag === 'hr') {
       out += '\n\n---'
     } else if (tag === 'table') {
-      const trRe = /<tr[^>]*>([\s\S]*?)<\/tr>/gi
-      let tm, rows = []
-      while ((tm = trRe.exec(inner))) {
-        const cells = []
-        const tdRe = /<t[dh][^>]*>([\s\S]*?)<\/t[dh]>/gi
-        let cm
-        while ((cm = tdRe.exec(tm[1]))) cells.push(inlineMd(cm[1]).replace(/\|/g, '\\|'))
-        if (cells.length) rows.push(`| ${cells.join(' | ')} |`)
+      const rows = []
+      if (inner.includes('</tr')) {
+        const trRe = /<tr[^>]{0,1000}>([\s\S]*?)<\/tr>/gi
+        let tm
+        while ((tm = trRe.exec(inner))) {
+          const cells = []
+          if (tm[1].includes('</t')) {
+            const tdRe = /<t[dh][^>]{0,1000}>([\s\S]*?)<\/t[dh]>/gi
+            let cm
+            while ((cm = tdRe.exec(tm[1]))) cells.push(inlineMd(cm[1], depth + 1).replace(/\|/g, '\\|'))
+          }
+          if (cells.length) rows.push(`| ${cells.join(' | ')} |`)
+        }
       }
       if (rows.length) {
         const header = rows[0]
@@ -572,36 +679,110 @@ export function blockMd(html) {
         out += `\n\n${rows.join('\n')}\n${sep}`
       }
     } else if (tag === 'a' || tag === 'strong' || tag === 'b' || tag === 'em' || tag === 'i') {
-      const t = inlineMd(inner)
+      const t = inlineMd(inner, depth + 1)
       if (t) out += t
     } else {
-      const t = blockMd(inner)
+      const t = blockMd(inner, depth + 1)
       if (t) out += t
     }
   }
   return img.restore(out)
 }
 
+// Read the content attribute of the FIRST <meta> whose property/name equals
+// `key`. Attribute order is arbitrary (content may precede property/name), and
+// the content value may itself contain quotes — locate the tag first, then
+// read content from inside it with a backreference pair.
+function metaContent(html, key) {
+  const tagM = new RegExp(`<meta\\b[^>]{0,1000}(?:property|name)=["']${key}["'][^>]{0,1000}>`, 'i').exec(html)
+  if (!tagM) return ''
+  const m = /content=("([^"]*)"|'([^']*)')/i.exec(tagM[0])
+  return m ? (m[2] || m[3] || '') : ''
+}
+
 // Page-level metadata the model actually asks about (who wrote it, when) —
-// cheap to harvest from <meta>, previously thrown away.
+// cheap to harvest from <meta>. publishedExplicit records whether the date
+// came from a semantic article/og key (authoritative) or a generic date/dc.date
+// stamp (often last-modified, weaker than a JSON-LD datePublished).
 function extractMeta(html) {
-  const get = (names) => {
+  const getKey = (names) => {
     for (const n of names) {
-      const m = new RegExp(`<meta[^>]+(?:property|name)=["']${n}["'][^>]+content=["']([^"']+)["']`, 'i').exec(html)
-      if (m) return decodeTextEntities(m[1]).replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim().slice(0, 80)
+      const v = metaContent(html, n)
+      if (v) {
+        return {
+          v: decodeTextEntities(v).replace(/<[^>]{1,1000}>/g, ' ').replace(/\s+/g, ' ').trim().slice(0, 80),
+          key: n,
+        }
+      }
     }
-    return ''
+    return { v: '', key: '' }
   }
+  const p = getKey(['article:published_time', 'og:published_time', 'pubdate', 'publishdate', 'date', 'dc.date'])
+  const a = getKey(['author', 'article:author', 'og:article:author'])
+  const d = getKey(['og:description', 'description'])
   return {
-    published: get(['article:published_time', 'og:published_time', 'pubdate', 'publishdate', 'date', 'dc.date']),
-    author: get(['author', 'article:author', 'og:article:author']),
-    description: get(['og:description', 'description']),
+    published: p.v,
+    publishedExplicit: /^(article|og):/i.test(p.key),
+    author: a.v,
+    description: d.v,
+  }
+}
+
+// schema.org JSON-LD blocks carry publication metadata (datePublished/author)
+// many news sites put nowhere else — <meta> covers only a subset and the
+// byline regex cannot see ISO-8601 timestamps. Parse the first ld+json block
+// that yields either field; tolerate broken JSON and every structural shape
+// (plain object, @type arrays, @graph graphs, author as string/object/array).
+// Bounded block size keeps a hostile giant script tag from dragging the scan.
+// 100k covers real article-list JSON-LD (news sites often embed dozens of
+// items); metadata sits in the first node anyway.
+const LDJSON_RE = /<script[^>]{0,1000}type=["']application\/ld\+json["'][^>]{0,1000}>([\s\S]{0,100000}?)<\/script\s*>/gi
+function jsonLdMeta(html) {
+  // Recursively collect every node: nested structures are common (ItemList →
+  // mainEntity → Article, WebPage → mainEntityOfPage), so a flat top-level
+  // scan alone misses most real payloads. Depth-capped against hostile
+  // nesting; arrays, @graph and plain object properties are all expanded.
+  const collect = (node, out, depth = 0) => {
+    if (!node || typeof node !== 'object' || depth > 20) return
+    if (Array.isArray(node)) {
+      for (const it of node) collect(it, out, depth + 1)
+      return
+    }
+    out.push(node)
+    if (Array.isArray(node['@graph'])) for (const it of node['@graph']) collect(it, out, depth + 1)
+    for (const k of Object.keys(node)) {
+      if (k === '@graph') continue
+      const v = node[k]
+      if (v && typeof v === 'object') collect(v, out, depth + 1)
+    }
+  }
+  const authorOf = (a) => {
+    if (typeof a === 'string') return a
+    if (Array.isArray(a)) return authorOf(a[0])
+    if (a && typeof a === 'object') return typeof a.name === 'string' ? a.name : authorOf(a.name)
+    return ''
+  }
+  const clean = (s) => (s ? decodeTextEntities(s).slice(0, 40) : '')
+  for (const m of html.matchAll(LDJSON_RE)) {
+    let json
+    try {
+      json = JSON.parse(m[1])
+    } catch {
+      continue
+    }
+    const nodes = []
+    collect(json, nodes)
+    for (const n of nodes) {
+      const published = clean(typeof n.datePublished === 'string' ? n.datePublished : '')
+      const author = clean(authorOf(n.author))
+      if (published || author) return { published, author }
+    }
   }
+  return { published: '', author: '' }
 }
 
-// Meta-less pages (measured: ruanyifeng.com carries ZERO author/date meta
-// tags) still print the byline inside the body head: "作者：阮一峰 日期：
-// 2026年8月21日". Fill ONLY the missing fields, and only from the first
+// Meta-less pages still print the byline inside the body head: "作者：阮一峰
+// 日期：2026年8月21日". Fill ONLY the missing fields, and only from the first
 // 600 chars of body text — deeper matches are content, not bylines.
 const BYLINE_AUTHOR_RE = /(?:作者|責任編輯|責任主编|editor)[：:\s]{1,3}([^\s，,。/|()（）[\]]{2,24})/
 const BYLINE_DATE_RE = /(?:发表日期|发布日期|发表时间|发布时间|日期|发表于|发布于|published(?:\s+on)?)[：:\s]{0,3}(\d{4}[-/.年]\s?\d{1,2}[-/.月]\s?\d{1,2}日?|\d{4}-\d{2}-\d{2}(?:[T ]\d{2}:\d{2})?)/i
@@ -621,9 +802,11 @@ function bylineFallback(text, meta) {
 }
 
 export function extract(html, mode) {
-  const titleMatch = /<title[^>]*>([\s\S]*?)<\/title>/i.exec(html) || /<meta[^>]+property=["']og:title["'][^>]+content=["']([^"']+)["']/i.exec(html)
-  const siteName = /<meta[^>]+property=["']og:site_name["'][^>]+content=["']([^"']+)["']/i.exec(html)
-  const langMatch = /<html[^>]+lang=["']([\w-]+)["']/i.exec(html)
+  html = defuseLt(html) // kills the no-'>'-tail quadratic scans before any picker runs
+  const titleTag = /<title[^>]{0,1000}>([\s\S]*?)<\/title>/i.exec(html)
+  const ogTitle = metaContent(html, 'og:title')
+  const siteName = metaContent(html, 'og:site_name')
+  const langMatch = /<html[^>]{1,1000}lang=["']([\w-]+)["']/i.exec(html)
   let main = stripNoise(pickMain(html))
   main = stripNoise(revealEscapedTags(main))
   let bodyText
@@ -633,6 +816,14 @@ export function extract(html, mode) {
     bodyText = textOnly(main).replace(/ +/g, ' ').trim()
   }
   const meta = extractMeta(html)
+  const ld = jsonLdMeta(html)
+  // Merge chain: <meta> → JSON-LD → byline. An explicit article:/og: meta date
+  // stays authoritative; otherwise a JSON-LD datePublished beats the generic
+  // date meta key (last-modified stamps are not publication times).
+  const published = ld.published
+    ? (meta.publishedExplicit ? meta.published : ld.published)
+    : meta.published
+  const author = ld.author || meta.author
   // Empty-body pages (login walls, JS-only shells) still carry og:description
   // — surface it as a fallback hint instead of nothing.
   let text = bodyText
@@ -641,10 +832,10 @@ export function extract(html, mode) {
     text = meta.description
     usedDescription = true
   }
-  const byline = bylineFallback(text, meta)
+  const byline = bylineFallback(text, { published, author })
   return {
-    title: titleMatch ? textOnly(titleMatch[1]) || titleMatch[1].trim() : '',
-    siteName: siteName ? siteName[1] : '',
+    title: titleTag ? textOnly(titleTag[1]) || titleTag[1].trim() : ogTitle,
+    siteName,
     lang: langMatch ? langMatch[1] : '',
     published: byline.published,
     author: byline.author,
@@ -701,7 +892,19 @@ export function smartTruncate(text, maxChars, offset = 0) {
     if (acc.length + p.length + (acc ? 2 : 0) > maxChars) break
     acc += (acc ? '\n\n' : '') + p
   }
-  if (!acc && maxChars > 0) acc = text.slice(offset, offset + maxChars)
+  if (!acc && maxChars > 0) {
+    // The first paragraph after the offset is itself longer than maxChars.
+    // Keep the alignment promise at sentence level before hard-slicing.
+    const p = paragraphs[start] || ''
+    const rel = Math.max(0, offset - pos)
+    const rest = p.length > rel ? p.slice(rel) : text.slice(offset)
+    const sentences = rest.match(/[^.!?\n。！？]+[.!?\n。！？]*/g) || [rest]
+    for (const s of sentences) {
+      if (acc.length + s.length > maxChars) break
+      acc += s
+    }
+    if (!acc) acc = rest.slice(0, maxChars)
+  }
   return {
     text: acc,
     truncated: offset + acc.length < total,
@@ -711,12 +914,38 @@ export function smartTruncate(text, maxChars, offset = 0) {
   }
 }
 
+// <base href> overrides the document base for relative URL resolution (old
+// forums / frame sites). Per HTML spec only the FIRST <base> in <head> counts;
+// attribute order is arbitrary and the href may be protocol-relative. Empty
+// href (= document URL) is a no-op; non-http(s) targets are rejected. The
+// head scan is bounded — the tag cannot legally appear beyond it.
+function detectBaseHref(html, finalUrl) {
+  const headEnd = html.search(/<\/head\s*>/i)
+  const head = headEnd >= 0 ? html.slice(0, headEnd) : html.slice(0, 16384)
+  const base = /<base\b[^>]{0,1000}href=["']([^"']+)["'][^>]{0,1000}>/i.exec(head)
+  if (!base) return finalUrl
+  const href = base[1].trim()
+  if (!href) return finalUrl
+  try {
+    const u = new URL(href, finalUrl)
+    return u.protocol === 'http:' || u.protocol === 'https:' ? u.href : finalUrl
+  } catch {
+    return finalUrl
+  }
+}
+
 function extractLinks(html, limit, baseUrl) {
   const links = []
   const seen = new Set()
   // Accept relative hrefs too — most real pages link internally with relative
   // paths — and resolve them against the page URL so the model can follow them.
-  const re = /<a[^>]+href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi
+  // A declared <base href> wins over the document URL (frame-site links would
+  // otherwise land on the wrong host).
+  const linkBase = detectBaseHref(html, baseUrl)
+  // Attribute runs bounded: an unclosed <a whose '>' is far away (or missing)
+  // must not make the regex scan the rest of the string at every anchor
+  // position. Real attributes never approach 1000 chars.
+  const re = /<a\b[^>]{0,1000}href=["']([^"']+)["'][^>]{0,1000}>([\s\S]*?)<\/a\s*>/gi
   let m
   // Iteration cap: a page whose nav repeats the same links thousands of times
   // would otherwise scan the whole HTML to fill `limit` unique entries. Scan
@@ -728,10 +957,14 @@ function extractLinks(html, limit, baseUrl) {
     if (!href || /^(javascript|mailto|tel|data):/i.test(href)) continue
     let url
     try {
-      url = baseUrl ? new URL(href, baseUrl).href : new URL(href).href
+      url = linkBase ? new URL(href, linkBase).href : new URL(href).href
     } catch {
       continue
     }
+    // Only http(s) links are followable by the model / crawler; other schemes
+    // (ftp:, data:) survive URL resolution and would pollute link lists and
+    // crawl queues.
+    if (!/^https?:/i.test(url)) continue
     // dedupe: nav bars repeat the same links many times — one entry per URL
     // keeps the link list compact (tokens) without losing coverage
     if (seen.has(url)) continue
@@ -755,26 +988,32 @@ function hostOf(url) {
 // the model had to notice the truncation and re-call the tool per page. The
 // recognition is deliberately conservative: rel=next (standard) or a short
 // anchor whose whole text is a next-page marker — no fuzzy guessing.
-const NEXT_TEXT_RE = /^(?:下一页|下页|后一页|下一頁|下頁|next page|next|›{1,3}|»{1,3}|>|older entries)$/i
+const NEXT_TEXT_RE = /^(?:[›»>]+\s*)?(?:下一页|下页|下一篇|下一頁|下頁|后一页|後一頁|next page|next|older entries|[›»>]{1,3})(?:\s*[›»>]+)?$/i
 
 export function findNextLink(html, baseUrl) {
+  // Same <base href> rule as extractLinks: frame/forum sites pin their
+  // document base, and a "next page" link resolves against it.
+  const base = detectBaseHref(html || '', baseUrl)
   const resolve = (href) => {
     try {
-      const u = new URL(href, baseUrl)
+      const u = new URL(href, base)
       return u.protocol === 'http:' || u.protocol === 'https:' ? u.href : null
     } catch {
       return null
     }
   }
-  const rel = /<(?:link|a)\b[^>]+rel=["']next["'][^>]*>/i.exec(html || '')
+  const rel = /<(?:link|a)\b[^>]{0,1000}rel=["']next["'][^>]{0,1000}>/i.exec(html || '')
   if (rel) {
     const href = /href=["']([^"']+)["']/i.exec(rel[0])
     const r = href && resolve(href[1])
     if (r) return r
   }
-  const re = /<a\b[^>]+href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a\s*>/gi
+  const re = /<a\b[^>]{0,1000}href=["']([^"']+)["'][^>]{0,1000}>([\s\S]*?)<\/a\s*>/gi
   let m
-  while ((m = re.exec(html || ''))) {
+  let scans = 0
+  // Bounded like extractLinks: a nav bar with thousands of anchors must not
+  // scan the whole DOM hunting for a next-marker.
+  while ((m = re.exec(html || '')) && scans++ < 400) {
     const t = textOnly(m[2])
     if (t.length <= 16 && NEXT_TEXT_RE.test(t)) {
       const r = resolve(m[1])
@@ -822,6 +1061,105 @@ function sliceFrom(full, offset, maxChars) {
   return out
 }
 
+// ---- JSON payloads: compact render ----
+// Indented JSON spends a newline + indent per key — pure token overhead for
+// the model. Render compact, and clip monster string values (embedded HTML,
+// base64 blobs) with a visible marker so the model knows there is more.
+const JSON_STR_MAX = 1500
+export function compactJson(value) {
+  const clip = (s) =>
+    s.length > JSON_STR_MAX ? s.slice(0, JSON_STR_MAX) + `…[+${s.length - JSON_STR_MAX} chars]` : s
+  const walk = (v, depth) => {
+    if (typeof v === 'string') return clip(v)
+    if (Array.isArray(v)) return v.map((x) => walk(x, depth + 1))
+    if (v && typeof v === 'object' && depth < 12) {
+      const o = {}
+      for (const k of Object.keys(v)) o[k] = walk(v[k], depth + 1)
+      return o
+    }
+    return v
+  }
+  try {
+    return JSON.stringify(walk(value, 0))
+  } catch {
+    // Extreme nesting/size where even native stringify exhausts the stack —
+    // degrade to a clipped raw render instead of crashing the tool call.
+    try {
+      return clip(String(value))
+    } catch {
+      return '[unserializable json]'
+    }
+  }
+}
+
+// ---- meta-refresh shells: follow to the real page ----
+// <meta http-equiv="refresh" content="0; url=..."> stubs (link hops,
+// anti-hotlink relays, no-JS SPA entries) serve only a jump page — the body
+// lives at the target. Parse the FIRST matching <meta>; attribute order and
+// case are arbitrary, content quotes are stripped, a protocol-relative target
+// resolves against the document (or <base href>). Only IMMEDIATE redirects
+// are followed: a timed auto-refresh (content="30; url=…") is a normal page
+// with content, not a shell. A bare timed refresh with no url part reloads
+// the same page — null. Non-http(s) targets are refused.
+export function metaRefreshTarget(html, baseUrl) {
+  // Locate the refresh <meta> tag first (any attribute order / case), then
+  // read content from inside that tag — a forward regex would demand
+  // http-equiv before content and miss the reversed order.
+  const re = /<meta\b[^>]{0,1000}(?:http-equiv|name)=["']refresh["'][^>]{0,1000}>/gi
+  let m
+  while ((m = re.exec(html || ''))) {
+    const tag = m[0]
+    const contentM = /content=("([^"]*)"|'([^']*)')/i.exec(tag)
+    if (!contentM) continue // a refresh meta without content is inert
+    const content = (contentM[2] || contentM[3] || '').trim()
+    // content="5" | content="0; url=http://…" | content=";url='…'" — the url
+    // part may be single/double quoted or bare; "url=" must be a real token.
+    const urlM = /\burl\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s;]+))/i.exec(content)
+    if (!urlM) return null
+    const delayM = /^\s*(\d+(?:\.\d+)?)/.exec(content)
+    if (delayM && Number(delayM[1]) > 0) return null // timed refresh: keep the page
+    const target = (urlM[1] || urlM[2] || urlM[3] || '').trim()
+    if (!target) return null
+    try {
+      const u = new URL(target, detectBaseHref(html || '', baseUrl))
+      return u.protocol === 'http:' || u.protocol === 'https:' ? u.href : null
+    } catch {
+      return null
+    }
+  }
+  return null
+}
+
+// Bounded following: at most 3 hops, deduped by normalized URL. Any fetch
+// problem (network error, non-HTML body) FAILS OPEN — the shell's own HTML
+// is kept for static extraction instead of erroring the call. Followed pages
+// are never written to the session cache under their own URL: the caller's
+// cache key is the ORIGINAL url, and a shell re-request should re-follow
+// rather than serve a foreign page.
+const MAX_META_REFRESH_HOPS = 3
+async function followMetaRefresh(html, finalUrl, externalSignal, cfg) {
+  let cur = html
+  let curUrl = finalUrl
+  let curCharset = '' // set only when a hop actually happened
+  const seen = new Set([normalizeUrl(finalUrl)])
+  for (let i = 0; i < MAX_META_REFRESH_HOPS; i++) {
+    const target = metaRefreshTarget(cur, curUrl)
+    if (!target) break
+    const norm = normalizeUrl(target)
+    if (!norm || seen.has(norm)) break
+    seen.add(norm)
+    const page = await fetchPage(target, externalSignal, cfg)
+    if (page.error) break
+    const ct = (page.contentType || '').split(';')[0].toLowerCase().trim()
+    if (!/html|xhtml/.test(ct)) break
+    const decoded = decodeBuffer(page.buffer, page.contentType)
+    cur = decoded.text
+    curCharset = decoded.charset
+    curUrl = page.finalUrl || target
+  }
+  return { html: cur, finalUrl: curUrl, charset: curCharset }
+}
+
 export async function readUrl(args, ctx, externalSignal, cfg = DEFAULTS) {
   const url = String((args && args.url) || '').trim()
   if (!/^https?:\/\//i.test(url)) return { error: 'Only http/https URLs are supported' }
@@ -842,22 +1180,21 @@ export async function readUrl(args, ctx, externalSignal, cfg = DEFAULTS) {
     /* keep raw url on parse failure */
   }
   const cacheKey = `${cacheUrl}|${mode}|${args.includeLinks === true ? 'links' : 'no-links'}`
+  // Failed fetches are cached briefly so the model doesn't re-request a
+  // broken URL in a loop (token + latency saver).
+  const fh = failCache.get(cacheKey)
+  if (fh) {
+    if (Date.now() - fh.time < 30000) return { error: fh.error, cached: true }
+    failCache.delete(cacheKey)
+  }
   const hit = cache.get(cacheKey)
-  if (hit) {
-    // Failed fetches are cached briefly so the model doesn't re-request a
-    // broken URL in a loop (token + latency saver).
-    if (hit.error) {
-      if (Date.now() - hit.time < 30000) return { error: hit.error, cached: true }
-      cache.delete(cacheKey)
-    } else if (Date.now() - hit.time < cfg.cacheTtlMs) {
-      const sliced = sliceFrom(hit.full, offset, maxChars)
-      return { ...sliced, cached: true }
-    }
+  if (hit && Date.now() - hit.time < cfg.cacheTtlMs) {
+    const sliced = sliceFrom(hit.full, offset, maxChars)
+    return { ...sliced, cached: true }
   }
 
   const failWith = (error) => {
-    if (cache.size >= cfg.cacheMax) cache.delete(cache.keys().next().value)
-    cache.set(cacheKey, { time: Date.now(), error })
+    cacheFail(cacheKey, error)
     return { error }
   }
 
@@ -870,7 +1207,7 @@ export async function readUrl(args, ctx, externalSignal, cfg = DEFAULTS) {
       const t = viaSeam.html.trim()
       if (t.startsWith('{') || t.startsWith('[')) {
         try {
-          return { kind: 'json', text: JSON.stringify(JSON.parse(t), null, 1) }
+          return { kind: 'json', text: compactJson(JSON.parse(t)) }
         } catch { /* not JSON — keep HTML pipeline */ }
       }
       return { kind: 'html', html: viaSeam.html, finalUrl: viaSeam.finalUrl, charset: 'provider-decoded' }
@@ -881,7 +1218,7 @@ export async function readUrl(args, ctx, externalSignal, cfg = DEFAULTS) {
     const decoded = decodeBuffer(page.buffer, page.contentType)
     if (/json$/.test(ct)) {
       try {
-        return { kind: 'json', text: JSON.stringify(JSON.parse(decoded.text), null, 1) }
+        return { kind: 'json', text: compactJson(JSON.parse(decoded.text)) }
       } catch { /* invalid JSON — serve through the HTML pipeline as text */ }
     }
     if (/[+/]xml$/.test(ct)) {
@@ -899,8 +1236,7 @@ export async function readUrl(args, ctx, externalSignal, cfg = DEFAULTS) {
 
   if (payload.kind === 'json') {
     const full = { url, title: '', siteName: hostOf(url), lang: '', charset: 'json', mode: 'json', fullText: payload.text, paginated: 1 }
-    if (cache.size >= cfg.cacheMax) cache.delete(cache.keys().next().value)
-    cache.set(cacheKey, { time: Date.now(), full })
+    cacheStore(cacheKey, full, cfg.cacheMax)
     return sliceFrom(full, offset, maxChars)
   }
   if (payload.kind === 'feed') {
@@ -910,22 +1246,34 @@ export async function readUrl(args, ctx, externalSignal, cfg = DEFAULTS) {
       mode: 'feed', fullText: f.text, paginated: 1, feedCount: f.count,
       links: args.includeLinks === true ? f.items : undefined,
     }
-    if (cache.size >= cfg.cacheMax) cache.delete(cache.keys().next().value)
-    cache.set(cacheKey, { time: Date.now(), full })
+    cacheStore(cacheKey, full, cfg.cacheMax)
     return sliceFrom(full, offset, maxChars)
   }
 
   let html = payload.html
   let finalUrl = payload.finalUrl || url
-  const charset = payload.charset
+  let charset = payload.charset
+
+  // meta-refresh shells (link hops, anti-hotlink relays, no-JS SPA entries):
+  // follow immediately, BEFORE extraction and SPA rendering — a shell that
+  // redirects needs no headless round-trip, and extracting the stub is wasted
+  // work. Fails open on any fetch problem; the follow is hop-bounded.
+  const followed = await followMetaRefresh(html, finalUrl, externalSignal, cfg)
+  html = followed.html
+  finalUrl = followed.finalUrl
+  // The target may use a different encoding than the shell page — reflect it.
+  if (followed.charset) charset = followed.charset
 
   let extracted = extract(html, mode)
   // Optional readability upgrade: cleaner article extraction when installed.
   const ext = await getReadabilityExtractor()
-  const upgrade = (target, htmlText) => {
+  // The base passed to readability must be the CURRENT final URL — after a
+  // meta-refresh follow or SPA render the document may live on another host,
+  // and relative links/images would resolve against the original request URL.
+  const upgrade = (target, htmlText, base) => {
     if (!ext) return target
     try {
-      const clean = ext(htmlText, url)
+      const clean = ext(htmlText, base || finalUrl)
       if (clean && clean.length > 0) {
         target.text = mode === 'markdown'
           ? blockMd(clean).replace(/\n{3,}/g, '\n\n').trim()
@@ -942,9 +1290,9 @@ export async function readUrl(args, ctx, externalSignal, cfg = DEFAULTS) {
   // extraction found almost nothing, try headless rendering (playwright).
   // Beyond script-heavy shells (looksLikeSpa, ≥5 <script> tags), a page whose
   // extraction is EMPTY but carries any <script> may be a JS-redirect shell
-  // (measured: taptap.cn — 2 scripts, empty <body>, redirects via JS) — with
-  // no text to lose, rendering is always worth one bounded attempt. Pages
-  // with no scripts at all cannot change under rendering and are skipped.
+  // (2 scripts, empty <body>, redirect issued from JS) — with no text to
+  // lose, rendering is always worth one bounded attempt. Pages with no
+  // scripts at all cannot change under rendering and are skipped.
   let rendered = false
   let spaHint = ''
   let renderedHtml = null
@@ -956,19 +1304,18 @@ export async function readUrl(args, ctx, externalSignal, cfg = DEFAULTS) {
       renderedHtml = rr.html
       // A challenge interstitial (Cloudflare "Just a moment...") must never
       // be accepted as an improvement — it can carry MORE text than the real
-      // static body (measured: 259 chars of verification prose vs 135 chars
-      // of real content on ruanyifeng.com) and would evict the latter.
+      // static body and would evict the latter.
       if (looksLikeChallenge(rr.html)) {
         renderedHtml = null // pagination/links must not read the interstitial DOM
         spaHint = extracted.text
           ? '已渲染但被人机验证页拦截，保留静态提取结果'
           : '已渲染但被人机验证页拦截（无静态内容可用）'
       } else {
-        const r2 = upgrade(extract(rr.html, mode), rr.html)
+        const r2 = upgrade(extract(rr.html, mode), rr.html, rr.finalUrl || finalUrl)
         // Accept the rendered text when it meaningfully beats the static one.
         // From an EMPTY static extraction even a short body is a real gain
-        // (measured: taptap-style JS shells render 30-60 chars of real content
-        // that the fixed +50 margin would reject).
+        // (JS shells can render just 30-60 chars of real content, below the
+        // fixed +50 margin).
         const gain = r2.text.length - extracted.text.length
         const accept = r2.text && (gain > 50 || (!extracted.text && r2.text.length >= 20))
         if (accept) {
@@ -1030,8 +1377,7 @@ export async function readUrl(args, ctx, externalSignal, cfg = DEFAULTS) {
     links: args.includeLinks === true ? extractLinks(renderedHtml || html, cfg.maxLinks, finalUrl) : undefined,
   }
 
-  if (cache.size >= cfg.cacheMax) cache.delete(cache.keys().next().value)
-  cache.set(cacheKey, { time: Date.now(), full })
+  cacheStore(cacheKey, full, cfg.cacheMax)
   return sliceFrom(full, offset, maxChars)
 }
 
@@ -1045,22 +1391,24 @@ function renderResult(value) {
   const meta = []
   if (r.siteName && r.siteName !== host) meta.push(r.siteName)
   if (r.lang) meta.push(r.lang)
-  if (r.charset && r.charset !== 'json' && r.charset !== 'feed') meta.push(`charset ${r.charset}`)
+  if (r.charset && r.charset !== 'utf-8' && r.charset !== 'json' && r.charset !== 'feed') {
+    meta.push(`charset ${r.charset}`)
+  }
   if (r.published) meta.push(r.published)
   if (r.author) meta.push(`by ${r.author}`)
   if (r.feedCount) meta.push(`feed · ${r.feedCount} 条`)
   if (r.mode === 'json') meta.push('json')
   if (meta.length) lines.push(meta.join(' · '))
-  if (r.charsStart > 0) {
-    lines.push(`(chars ${r.charsStart}+${r.charsReturned}/${r.charsTotal} — offset 续读)`)
-  } else if (r.truncated) {
-    lines.push(`(chars ${r.charsReturned}/${r.charsTotal} — 截断，offset 续读)`)
-  } else if (typeof r.charsTotal === 'number') {
-    lines.push(`(chars ${r.charsTotal})`)
-  }
-  if (r.cached) lines.push('(cached)')
-  if (r.paginated > 1) lines.push(`(已自动拼接 ${r.paginated} 页)`)
-  if (r.rendered) lines.push('(rendered — JS 执行后内容)')
+  // One compact status line instead of several parenthetical lines — the
+  // flags are hints, not content, so they share a single line of tokens.
+  const flags = []
+  if (r.charsStart > 0) flags.push(`chars ${r.charsStart}+${r.charsReturned}/${r.charsTotal}`, 'offset 续读')
+  else if (r.truncated) flags.push(`chars ${r.charsReturned}/${r.charsTotal}`, '截断', 'offset 续读')
+  else if (typeof r.charsTotal === 'number') flags.push(`chars ${r.charsTotal}`)
+  if (r.cached) flags.push('cached')
+  if (r.paginated > 1) flags.push(`已拼接${r.paginated}页`)
+  if (r.rendered) flags.push('rendered')
+  if (flags.length) lines.push(`(${flags.join(' · ')})`)
   if (!r.text) {
     if (r.charsStart > 0 && !r.truncated) {
       lines.push('(offset 已到文末，无更多内容)')
@@ -1134,6 +1482,11 @@ function readLinksTool(ctx, cfg) {
         html = decodeBuffer(page.buffer, page.contentType).text
         finalUrl = page.finalUrl || url
       }
+      // Same meta-refresh shell following as read_url: a link hop serves a
+      // stub whose real links live at the target. Fails open, hop-bounded.
+      const followed = await followMetaRefresh(html, finalUrl, exec && exec.signal, cfg)
+      html = followed.html
+      finalUrl = followed.finalUrl
       let links = extractLinks(html, limit, finalUrl)
       // SPA fallback: a JS-only page yields no links from its static HTML;
       // render it and re-extract when the static result looks empty. Any
@@ -1281,7 +1634,7 @@ function sameHost(a, b) {
 }
 
 // URLs not worth crawling: static assets, login/auth paths, feeds, sitemaps.
-const NOISE_EXT = /\.(png|jpe?g|gif|svg|webp|ico|bmp|css|js|json|xml|pdf|zip|gz|tar|7z|mp3|mp4|avi|mov|webm|woff2?|ttf|eot|map)(\?|#|$)/i
+const NOISE_EXT = /\.(png|jpe?g|gif|svg|webp|ico|bmp|css|js|json|xml|pdf|zip|gz|tar|7z|mp3|m3u8?|mpd|flv|ts|mp4|avi|mov|webm|woff2?|ttf|eot|map)(\?|#|$)/i
 const NOISE_PATH = /(\/login|\/signin|\/register|\/logout|\/signup|\/api\/|\/admin|\/wp-admin|\/wp-login|\/feed|\/rss|\/sitemap|\/robots\.txt|\/cdn-cgi)/i
 function isNoiseUrl(url) {
   return NOISE_EXT.test(url) || NOISE_PATH.test(url)
@@ -1453,6 +1806,7 @@ export function apply(ctx, config) {
   // Temporal composability: unload must fully revert side effects.
   ctx.effect(() => () => {
     cache.clear()
+    failCache.clear()
     closeBrowser().catch(() => {})
   })